<commit_before>#! /usr/bin/env python

import datetime
import string
import logging
try: import simplejson as json
except ImportError: import json

from lxml import html
from lxml.html.clean import Cleaner
from collections import deque

from lib.constants import NFL as nfl
from lib.utils import Utils as utils

from google.appengine.api import mail
from google.appengine.ext import webapp

from models.score import ScoreFactory
from models.spread import SpreadFactory

from google.appengine.ext.webapp.mail_handlers import InboundMailHandler 
from google.appengine.ext.webapp.util import run_wsgi_app

class ReceiveMail(InboundMailHandler):
    __SENDER = ''

    """
    """

    def receive(self, message):
        """Event that is fired upon receiving an email

        """
        result = []

        try:
            spread_data = self._parse_for_spread_data(message)
            player_picks = self._map_data_to_dict(spread_data)

            for name, pick in self._convert_to_id_table(player_picks).iteritems():
                result.append(self._convert_to_spread_object(name, pick))
        except Exception, e:
            logging.error(e)
            result = [e]

        if len(result) > 0:
            spread = SpreadFactory().get_instance()
            spread.save(utils.default_week(), result)
            self._success(message.to, message.sender, message.subject, result)
            #self._debug_message(message.to, message.sender, message.subject, result)

        
    def _parse_for_spread_data(self, message):
        htmltext = message.bodies('text/html')
        tree = None
        tables = None
        cleaner = Cleaner(style=True,
                          links=True,
                          page_structure=True,
                          javascript=True,
                          scripts=True,
                          remove_tags=(['font','div', 'u', 'b', 'i']))
        result = []

        for content_type, body in htmltext:
            decoded = body.decode()

            tree = self._filter_and_transform_to_tree(decoded)

            tables = tree.xpath('//tbody')
            for page in tables:
                spread_data = []

                for row in cleaner.clean_html(page):
                    cell_row_data = self._extract_elements_from_tr(row)

                    if len(cell_row_data) > 1:
                        if 'DEFAULT PICK' in cell_row_data[0] or 'POINTS' in cell_row_data[0]:
                            # no one cares about season meta-data
                            pass
                        elif 'Name' in cell_row_data[0] and len(spread_data) > 0:
                            # Came across a new page
                            result.append(spread_data)

                            # Reset the spread_data before saving to it again
                            spread_data = []
                            spread_data.append(cell_row_data)
                        else:
                            spread_data.append(cell_row_data)


                result.append(spread_data)
                spread_data = []

        return result

    def _map_data_to_dict(self, spread_data):
        result = {}

        for page in spread_data:
            width = 0
            current = {}
            current_players = []

            for row in page:
                working = deque(row)

                # Check if working with header
                if 'Name' in working[0]:
                    #drop meta-header
                    working.popleft()

                    # Save header data as keys
                    while len(working) > 0:
                        name = working.popleft()

                        # Guard against placeholder columns
                        if 'X' != name and 'x' != name:
                            current[name] = []
                            current_players.append(name)
                            width += 1
                else:
                    while len(working) > width:
                        # remove extraneous data from right-end of row
                        working.pop()

                    for player in current_players:
                        current[player].append(working.popleft())

            # combine current with final result
            result.update(current)

        return result


    def _convert_to_id_table(self, player_picks):
        mapping = self._get_game_ids_map()
        result = {}
        picks = []

        for player_name in player_picks:
            picks = deque(player_picks[player_name])
            canary = 1000

            result[player_name] = {}
            while len(picks) > 0 and canary > 0:
                item = []
                # get the game data
                if len(picks) > 2:
                    try:
                        float(picks[2])

                        # Spread, Over/Under, & Total Score
                        for i in range(0,3):
                            item.append(picks.popleft())
                    except ValueError:
                        # Single-game pick
                        item.append(picks.popleft())
                else:
                    item.append(picks.popleft())

                # Guard against scratched picks
                if item[0] != 'X' and item[0] != 'x':
                    # match to id
                    try:
                        game_id = unicode(mapping[item[0]])
                    except KeyError:
                        logging.error('KeyError during mapping')
                        logging.error(player_name)
                        logging.error(player_picks)

                        self._ping('Error Received', {
                                "Player Name": player_name,
                                "Player Picks": picks
                            });

                        # Bail from error
                        return []

                    # Save<commit_msg>Better logging during a KeyMap error<commit_after>#! /usr/bin/env python

import datetime
import string
import logging
try: import simplejson as json
except ImportError: import json

from lxml import html
from lxml.html.clean import Cleaner
from collections import deque

from lib.constants import NFL as nfl
from lib.utils import Utils as utils

from google.appengine.api import mail
from google.appengine.ext import webapp

from models.score import ScoreFactory
from models.spread import SpreadFactory

from google.appengine.ext.webapp.mail_handlers import InboundMailHandler 
from google.appengine.ext.webapp.util import run_wsgi_app

class ReceiveMail(InboundMailHandler):
    __SENDER = ''

    """
    """

    def receive(self, message):
        """Event that is fired upon receiving an email

        """
        result = []

        try:
            spread_data = self._parse_for_spread_data(message)
            player_picks = self._map_data_to_dict(spread_data)

            for name, pick in self._convert_to_id_table(player_picks).iteritems():
                result.append(self._convert_to_spread_object(name, pick))
        except Exception, e:
            logging.error(e)
            result = [e]

        if len(result) > 0:
            spread = SpreadFactory().get_instance()
            spread.save(utils.default_week(), result)
            self._success(message.to, message.sender, message.subject, result)
            #self._debug_message(message.to, message.sender, message.subject, result)

        
    def _parse_for_spread_data(self, message):
        htmltext = message.bodies('text/html')
        tree = None
        tables = None
        cleaner = Cleaner(style=True,
                          links=True,
                          page_structure=True,
                          javascript=True,
                          scripts=True,
                          remove_tags=(['font','div', 'u', 'b', 'i']))
        result = []

        for content_type, body in htmltext:
            decoded = body.decode()

            tree = self._filter_and_transform_to_tree(decoded)

            tables = tree.xpath('//tbody')
            for page in tables:
                spread_data = []

                for row in cleaner.clean_html(page):
                    cell_row_data = self._extract_elements_from_tr(row)

                    if len(cell_row_data) > 1:
                        if 'DEFAULT PICK' in cell_row_data[0] or 'POINTS' in cell_row_data[0]:
                            # no one cares about season meta-data
                            pass
                        elif 'Name' in cell_row_data[0] and len(spread_data) > 0:
                            # Came across a new page
                            result.append(spread_data)

                            # Reset the spread_data before saving to it again
                            spread_data = []
                            spread_data.append(cell_row_data)
                        else:
                            spread_data.append(cell_row_data)


                result.append(spread_data)
                spread_data = []

        return result

    def _map_data_to_dict(self, spread_data):
        result = {}

        for page in spread_data:
            width = 0
            current = {}
            current_players = []

            for row in page:
                working = deque(row)

                # Check if working with header
                if 'Name' in working[0]:
                    #drop meta-header
                    working.popleft()

                    # Save header data as keys
                    while len(working) > 0:
                        name = working.popleft()

                        # Guard against placeholder columns
                        if 'X' != name and 'x' != name:
                            current[name] = []
                            current_players.append(name)
                            width += 1
                else:
                    while len(working) > width:
                        # remove extraneous data from right-end of row
                        working.pop()

                    for player in current_players:
                        current[player].append(working.popleft())

            # combine current with final result
            result.update(current)

        return result


    def _convert_to_id_table(self, player_picks):
        mapping = self._get_game_ids_map()
        result = {}
        picks = []

        for player_name in player_picks:
            picks = deque(player_picks[player_name])
            canary = 1000

            result[player_name] = {}
            while len(picks) > 0 and canary > 0:
                item = []
                # get the game data
                if len(picks) > 2:
                    try:
                        float(picks[2])

                        # Spread, Over/Under, & Total Score
                        for i in range(0,3):
                            item.append(picks.popleft())
                    except ValueError:
                        # Single-game pick
                        item.append(picks.popleft())
                else:
                    item.append(picks.popleft())

                # Guard against scratched picks
                if item[0] != 'X' and item[0] != 'x':
                    # match to id
                    try:
                        game_id = unicode(mapping[item[0]])
                    except KeyError:
                        logging.error('KeyError during mapping')
                        if len(item) > 0:
                            logging.error(item[0])
                        logging.error(player_name)
                        logging.error(player_picks)


                        # Bail from error
                        return []

                    # Save
{'fraction_non_alphanumeric': 0.07644120883772115, 'fraction_numerical': 0.004063320071108102, 'mean_word_length': 1.2645198389879242, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2183644', 'n_tokens_mistral': 3061, 'n_tokens_neox': 2836, 'n_words': 734}