<commit_before>"""
   Codechef Unofficial API.

   Parses all the user data in JSON.

   Author: <NAME>
           <EMAIL>


    TODO: Create a tester program, which can check if each method is working perfectly or not.
"""

from BeautifulSoup import BeautifulSoup
import os
from collections import OrderedDict
import json
import logging
import re


from utils import *
from problem import getProblemData
from contest import getContestList, getContestData

#
#	Helper function for getUserData() to parse the list of complete and partial problems.
#
def parseProblems(problemsC):
    problemDict = OrderedDict()

    for problemGroup in problemsC:
        problemList = []
        problemGroupList = problemGroup.findAll('a')
        for problem in problemGroupList:
            problemList.append(problem.text)
        
        #Added arbitary contest code "PRACTICE" for practice problems.
        problemDict[ "PRACTICE" if problemGroup.b.text.startswith("Practice") else problemGroup.b.text ] = problemList

    return problemDict

"""
	getUserData() does all the dirty work of parsing the HTML and junxing it altogether
	in a crude 'attributes' dict.
"""

# TODO : Try to parse the SVG image of the rating curve, to extract all data about the contest rating at any time.
# REFACTOR : Split all the parsing methods into seperate, for easy debugging.

def getUserData(username , expiryTime = 0, writeInFile = False):

    # Dictionary returning all the scraped data from the HTML.    
    attributes = OrderedDict()

    #
    #   Firstly, checking if the userdata is already stored in a CJSON file 
    #   which is not yet expired.
    #
    if expiryTime > 0:
        attributes = checkInFile('users/' + username, expiryTime)

        #   Returning the attributes object.
        if attributes is not None:
            print '[*] Found file. Returning object.'
            return attributes
        else:
            attributes = OrderedDict()

    soup = downloadUserPage(username)

    # The profile_tab contains all the data about the user.
    profileTab = soup.find('div', {'class': 'profile'})

    #   This profile consists of four tables.
    #
    #   ->  The first table just contains the real name of the user, and the display picture.
    #   ->  The second table contains the info about the user, and the problems info.
    #   ->  The third table contains the problem statistics.
    #   ->  The fourth table contains the performance graphs, in SVG format.
    #

    #The real name is present in a simple div.user-name-box,
    attributes['realName'] = profileTab.find('div', {'class' : 'user-name-box'}).text

    #The displayPicture link is present in div.user-thumb-pic
    attributes['displayPicture'] = profileTab.find('div', {'class' : 'user-thumb-pic'}).img['src']

    row = profileTab.table.findNext("table").tr
    #
    #	Parsing the personal data of the user.
    #
    while not row.text.startswith("Problems"):
        # Strip the text of unwanted &nbsp, and splitting via the :
        parsedText = row.text.replace("&nbsp;", '').split(':')
        attributes[ camelCase(parsedText[0]) ] = parsedText[1]
        row = row.findNext("tr")

    #
    #	Removing unwanted keys from attributes (for now)
    #<commit_msg>Added username as a key<commit_after>"""
   Codechef Unofficial API.

   Parses all the user data in JSON.

   Author: <NAME>
           <EMAIL>


    TODO: Create a tester program, which can check if each method is working perfectly or not.
"""

from BeautifulSoup import BeautifulSoup
import os
from collections import OrderedDict
import json
import logging
import re


from utils import *
from problem import getProblemData
from contest import getContestList, getContestData

#
#	Helper function for getUserData() to parse the list of complete and partial problems.
#
def parseProblems(problemsC):
    problemDict = OrderedDict()

    for problemGroup in problemsC:
        problemList = []
        problemGroupList = problemGroup.findAll('a')
        for problem in problemGroupList:
            problemList.append(problem.text)
        
        #Added arbitary contest code "PRACTICE" for practice problems.
        problemDict[ "PRACTICE" if problemGroup.b.text.startswith("Practice") else problemGroup.b.text ] = problemList

    return problemDict

"""
	getUserData() does all the dirty work of parsing the HTML and junxing it altogether
	in a crude 'attributes' dict.
"""

# TODO : Try to parse the SVG image of the rating curve, to extract all data about the contest rating at any time.
# REFACTOR : Split all the parsing methods into seperate, for easy debugging.

def getUserData(username , expiryTime = 0, writeInFile = False):

    # Dictionary returning all the scraped data from the HTML.    
    attributes = OrderedDict()

    #
    #   Firstly, checking if the userdata is already stored in a CJSON file 
    #   which is not yet expired.
    #
    if expiryTime > 0:
        attributes = checkInFile('users/' + username, expiryTime)

        #   Returning the attributes object.
        if attributes is not None:
            print '[*] Found file. Returning object.'
            return attributes
        else:
            attributes = OrderedDict()

    soup = downloadUserPage(username)

    # The profile_tab contains all the data about the user.
    profileTab = soup.find('div', {'class': 'profile'})

    #   This profile consists of four tables.
    #
    #   ->  The first table just contains the real name of the user, and the display picture.
    #   ->  The second table contains the info about the user, and the problems info.
    #   ->  The third table contains the problem statistics.
    #   ->  The fourth table contains the performance graphs, in SVG format.
    #

    #Add the username too, for convinenece.
    attributes['username'] = username

    #The real name is present in a simple div.user-name-box,
    attributes['realname'] = profileTab.find('div', {'class' : 'user-name-box'}).text

    #The displayPicture link is present in div.user-thumb-pic
    attributes['displayPicture'] = profileTab.find('div', {'class' : 'user-thumb-pic'}).img['src']

    row = profileTab.table.findNext("table").tr
    #
    #	Parsing the personal data of the user.
    #
    while not row.text.startswith("Problems"):
        # Strip the text of unwanted &nbsp, and splitting via the :
        parsedText = row.text.replace("&nbsp;", '').split(':')
        attributes[ camelCase(parsedText[0]) ] = parsedText[1]
        row = row.findNext("tr")

    #
    #	Removing unwanted keys from attributes (for now)
    #
{'fraction_non_alphanumeric': 0.08764215314632297, 'fraction_numerical': 0.0012130401819560274, 'mean_word_length': 2.9402628434886497, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3899060', 'n_tokens_mistral': 1873, 'n_tokens_neox': 1793, 'n_words': 698}