<commit_before>#!/usr/bin/python
"""
	Program Developed for the course: MultiDisciplinary Project

	The program get as a argument a String that corresponds to the username that 
	we want to looking for into StackOverflow. As a result we have a list formed
	by a tuple {user, id, total_score} which matched with the given String.

	@author: <NAME>.
"""

import urllib2
import re
import json
import sys
from bs4 import BeautifulSoup

# Auxiliar functions

def generateURL(username):
    return 'http://stackoverflow.com/users?tab=reputation/users/filter&search=' + username + '&filter=week&tab=reputation'

def getPageSourceCode(page):
    response = urllib2.urlopen(page)
    return response.read()

"""

Structure of the parse text:

# 1. user_id_1 		2. user_name_1

# 4. user_id_2 		5. user_name_2

"""
# Main Function

def searchUser(username):
    text = getPageSourceCode(generateURL(username))
    matches = re.finditer('<a href="/users/(\d+)/(\D+)">\D*</a>', text)
    final_list = []
    for match in matches:
        subtext = text[match.start():]
        total_score = re.search('total reputation: (\d+)', subtext).group(1)
        final_list.append({
            'user': match.group(2), 
            'id': match.group(1),
            'total_score': total_score
        })
    return final_list

def get_answers_page(user, number):
    return 'http://stackoverflow.com/users/{}/{}?tab=answers&page={}'.format(user['id'], user['user'], number)

def parse_answer_url(url):
    HTTP_STACKOVERFLOW = 'http://stackoverflow.com'
    text = getPageSourceCode(url)
    soup = BeautifulSoup(text)
    accepted = soup.find_all('div', {'class': 'answered-accepted'})
    urls = []
    for div in accepted:
        rem = 'window.location.href='
        url = div['onclick']
        url = url[len(rem) + 1:]
        url = HTTP_STACKOVERFLOW + url[:-1]
        urls.append(url)
        parse_question_score(url)
    return urls

def parse_question_score(url):
    text = getPageSourceCode(url)
    soup = BeautifulSoup(text)
    votes = soup.find_all('span', {'class', 'vote-count-post'})
    if len(votes) < 2:
        return {'question': 0, 'answer': 0}
    print int(votes[0].contents[0]), int(votes[1].contents[0])

if __name__ == '__main__':
    username = sys.argv[1]
    json_data = searchUser(username)
    user = json_data[0]
    answer_url = get_answers_page(user, 1)
    parse_answer_url(answer_url)
    # use dumps for pretty printing    
    print json.dumps(json_data, indent=4)
<commit_msg>Final alpha 1.<commit_after>#!/usr/bin/python
"""
	Program Developed for the course: MultiDisciplinary Project

	The program get as a argument a String that corresponds to the username that 
	we want to looking for into StackOverflow. As a result we have a list formed
	by a tuple {user, id, total_score} which matched with the given String.

	@author: <NAME>.
"""

import urllib
import urllib2
import re
import json
import sys
import calendar
from datetime import datetime
from bs4 import BeautifulSoup

# Auxiliar functions

def generateURL(username):
    username = urllib.quote(username)
    return 'http://stackoverflow.com/users?tab=reputation/users/filter&search=' + username + '&filter=week&tab=reputation'

def getPageSourceCode(page):
    response = urllib2.urlopen(page)
    return response.read()

"""

Structure of the parse text:

# 1. user_id_1 		2. user_name_1

# 4. user_id_2 		5. user_name_2

"""
# Main Function

def searchUser(username, id=None):
    text = getPageSourceCode(generateURL(username))
    text = BeautifulSoup(text)
    divs = text.find_all('div', {'class': 'user-details'})
    final_list = []
    for div in divs:
        a = div.find('a')
        username = a.contents[0]
        found_id = a['href'].split('/')[2]
        total_score = div.find('span', {'class': 'reputation-score'}).contents[0]
#    matches = re.finditer('<a href="/users/(\d+)/(\.+)">.*</a>', text)
        final_list.append({
            'user': username, 
            'id': found_id,
            'total_score': total_score
        })
    return final_list

def get_answers_page(user, number):
    username = urllib.quote(user['user'])
    return 'http://stackoverflow.com/users/{0}/{1}?tab=answers&page={2}'.format(user['id'], username, number)

def parse_answer_url(url):
    HTTP_STACKOVERFLOW = 'http://stackoverflow.com'
    text = getPageSourceCode(url)
    soup = BeautifulSoup(text)
    accepted = soup.find_all('div', {'class': 'answered-accepted'})
    urls = []
    for div in accepted:
        rem = 'window.location.href='
        url = div['onclick']
        url = url[len(rem) + 1:]
        url = HTTP_STACKOVERFLOW + url[:-1]
        urls.append(url)
        print json.dumps(parse_question_score(url), indent=4)
    return urls

def parse_question_score(url):
    DATE_2000 = 946684800
    text = getPageSourceCode(url)
    soup = BeautifulSoup(text)
    votes = soup.find_all('span', {'class': 'vote-count-post'})
    td_owner = soup.find('td', {'class': 'owner'})
    details = td_owner.find('div', {'class', 'user-details'})
    time = td_owner.find('div', {'class', 'user-action-time'}).span['title']
    dt = datetime.strptime(time, '%Y-%m-%d %H:%M:%SZ')
    ut = (calendar.timegm(dt.utctimetuple()) - DATE_2000) / 3600 / 24
    
    print ut, time
    splits = details.a['href'].split('/')
    id = int(splits[-2])
    questioner = searchUser(details.a.contents[0], id)[0]
    if len(votes) < 2:
        return {'question': 0, 'answer': 0}
    return {
        'question_score': int(votes[0].contents[0]),
        'questioner_reputation': questioner['total_score'],
        'answer_score': int(votes[1].contents[0])
    }	

if __name__ == '__main__':
    username = sys.argv[1]
    json_data = searchUser(username)
    user = json_data[0]
    i = 1
    while True:
        answer_url = get_answers_page(user, i)
        urls = parse_answer_url(answer_url)
        i = i + 1
        if len(urls) == 0:
            break;
    # use dumps for pretty printing    
    # print json.dumps(json_data, indent=4)

{'fraction_non_alphanumeric': 0.1431648715824358, 'fraction_numerical': 0.014415907207953604, 'mean_word_length': 3.28388928317956, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '783970', 'n_tokens_mistral': 2090, 'n_tokens_neox': 2020, 'n_words': 477}