<commit_before>__author__ = 'Greg'

import pandas as pd
import gzip
import numpy as np
import lxml.etree as et
import itertools


def recur_dictify(frame):
    """
    h/t: http://stackoverflow.com/a/19900276/843419
    :param frame: a pandas data frame
    :return: a nested dictionary with the columns as the keys and the final one
    as the value. Best if the keys are arranged and sorted and there are no
    duplicates.
    """
    if len(frame.columns) == 1:
        if frame.values.size == 1:
            return frame.values[0][0]
        return frame.values.squeeze()
    grouped = frame.groupby(frame.columns[0])
    d = {k: recur_dictify(g.ix[:, 1:]) for k, g in grouped}
    return d


def pick_county(dict_table, sctg, zone):
    """
    :param sctg: the commodity code for the truck's cargo
    :param dict_table: the appropriate lookup table
    :return: the O or D county FIPS code
    """
    # get the relevant county lookup table
    county = np.random.choice(
        dict_table[zone][sctg].keys(),
        p=dict_table[zone][sctg].values())
    return county


def pick_ienode(dict_table, mode, zone):
    """
    :param dict_table: the dictionary table of import and export nodes.
    :param mode: the import or export mode, indicating what facility (port,
      airport, seaport) should be chosen.
    :param zone: the FAF zone on the import or export end.
    :return: the airport, seaport, or highway crossing
    """
    ienode = np.random.choice(
        dict_table[zone][mode].keys(),
        p=dict_table[zone][mode].values())
    return ienode


def get_start_day():
    """
    :return: a random day of the week. For now all days are the same,
    but we don't have to make it that way. We have a two-week simulation
    """
    return np.random.randint(0, 364)


def get_departure_time():
    """
    :return: a random time in the day, bimodally distributed.
    """
    flag = np.random.binomial(1, 0.5)
    y = np.random.normal(9, 2.5) * (1 - flag) + \
        np.random.normal(16, 2.5) * flag
    if y < 0:
        # time cannot be less than midnight
        y = np.random.randint(0, 6 * 3600)
    elif y > 24 * 3600:
        # or greater than midnight
        y = np.random.randint(18 * 3600, 24 * 3600)
    else:
        y *= 3600
    return int(y)


def get_coord(name, dim):
    return str(FAC_COORDS[dim][name])

class TruckPlan:
    """Critical information for the truck plan"""
    id_iter = itertools.count(1)

    def __init__(self, origin, destination, sctg, inmode, outmode):
        # get the departure time ---
        self.time = None
        self.get_time()

        # only write the plan if the truck runs in the first week
        if self.time <= 7 * 24 * 3600:
            """
            :rtype : a truck plan with origin, destination, etc.
            """
            self.id = self.id_iter.next()
            self.origin = origin
            self.destination = destination
            self.sctg = sctg
            self.inmode = inmode
            self.outmode = outmode
    
            # get the origin points ----
            if self.inmode in ['1', '3', '4']:  # imported?
                try:
                    # If a valid import node exists, use it
                    self.origin = pick_ienode(EXIM_DICT, self.inmode, self.origin)
                except KeyError:
                    # If it doesn't, just assign like normal
                    self.get_origin()
            else:
                self.get_origin()
    
            # get the destination points ----
            if self.outmode in ['1', '3', '4']:  # imported?
                try:
                    # If a valid import node exists, use it
                    self.destination = pick_ienode(EXIM_DICT, self.outmode,
                                                   self.destination)
                except KeyError:
                    # If it doesn't, just assign like normal
                    self.get_destination()
            else:
                self.get_destination()
    
            self.write_plan()

    def display_plan(self):
        print "Origin: ", self.origin, "Destination", self.destination

    def get_origin(self):
        # Is the truck coming from Alaska?
        if self.origin == '20':
            # Is it going to states on the west coast?
            if self.destination in west_coast_f3z:
                # I-5 at the Washington/British Columbia border
                self.origin = '3004'
            else:
                # I-15 at the Montana/Alberta border
                self.origin = '3310'
        else:
            self.origin = pick_county(MAKE_DICT, self.sctg, self.origin)

    def get_destination(self):
        # Is the truck going to Alaska?
        if self.destination == '20':<commit_msg>Try to handle serialization error.<commit_after>__author__ = 'Greg'

import pandas as pd
import gzip
import numpy as np
import lxml.etree as et
import itertools


def recur_dictify(frame):
    """
    h/t: http://stackoverflow.com/a/19900276/843419
    :param frame: a pandas data frame
    :return: a nested dictionary with the columns as the keys and the final one
    as the value. Best if the keys are arranged and sorted and there are no
    duplicates.
    """
    if len(frame.columns) == 1:
        if frame.values.size == 1:
            return frame.values[0][0]
        return frame.values.squeeze()
    grouped = frame.groupby(frame.columns[0])
    d = {k: recur_dictify(g.ix[:, 1:]) for k, g in grouped}
    return d


def pick_county(dict_table, sctg, zone):
    """
    :param sctg: the commodity code for the truck's cargo
    :param dict_table: the appropriate lookup table
    :return: the O or D county FIPS code
    """
    # get the relevant county lookup table
    county = np.random.choice(
        dict_table[zone][sctg].keys(),
        p=dict_table[zone][sctg].values())
    return county


def pick_ienode(dict_table, mode, zone):
    """
    :param dict_table: the dictionary table of import and export nodes.
    :param mode: the import or export mode, indicating what facility (port,
      airport, seaport) should be chosen.
    :param zone: the FAF zone on the import or export end.
    :return: the airport, seaport, or highway crossing
    """
    ienode = np.random.choice(
        dict_table[zone][mode].keys(),
        p=dict_table[zone][mode].values())
    return ienode


def get_start_day():
    """
    :return: a random day of the week. For now all days are the same,
    but we don't have to make it that way. We have a two-week simulation
    """
    return np.random.randint(0, 364)


def get_departure_time():
    """
    :return: a random time in the day, bimodally distributed.
    """
    flag = np.random.binomial(1, 0.5)
    y = np.random.normal(9, 2.5) * (1 - flag) + \
        np.random.normal(16, 2.5) * flag
    if y < 0:
        # time cannot be less than midnight
        y = np.random.randint(0, 6 * 3600)
    elif y > 24 * 3600:
        # or greater than midnight
        y = np.random.randint(18 * 3600, 24 * 3600)
    else:
        y *= 3600
    return int(y)


def get_coord(name, dim):
    return str(FAC_COORDS[dim][name])

class TruckPlan:
    """Critical information for the truck plan"""
    id_iter = itertools.count(1)

    def __init__(self, origin, destination, sctg, inmode, outmode):
        # get the departure time ---
        self.time = None
        self.get_time()

        # only write the plan if the truck runs in the first week
        if self.time <= 7 * 24 * 3600:
            """
            :rtype : a truck plan with origin, destination, etc.
            """
            self.id = self.id_iter.next()
            self.origin = origin
            self.destination = destination
            self.sctg = sctg
            self.inmode = inmode
            self.outmode = outmode
    
            # get the origin points ----
            if self.inmode in ['1', '3', '4']:  # imported?
                try:
                    # If a valid import node exists, use it
                    self.origin = pick_ienode(EXIM_DICT, self.inmode, self.origin)
                except KeyError:
                    # If it doesn't, just assign like normal
                    self.get_origin()
            else:
                self.get_origin()
    
            # get the destination points ----
            if self.outmode in ['1', '3', '4']:  # imported?
                try:
                    # If a valid import node exists, use it
                    self.destination = pick_ienode(EXIM_DICT, self.outmode,
                                                   self.destination)
                except KeyError:
                    # If it doesn't, just assign like normal
                    self.get_destination()
            else:
                self.get_destination()
    
            # Write plan to xml document. This can fail with a serialization
            # error.
            try: 
                self.write_plan()
            except SerialisationError:
                self.display_plan()

    def display_plan(self):
        print "Id: ", self.id
        print "Origin: ", self.origin, "Destination", self.destination

    def get_origin(self):
        # Is the truck coming from Alaska?
        if self.origin == '20':
            # Is it going to states on the west coast?
            if self.destination in west_coast_f3z:
                # I-5 at the Washington/British Columbia border
                self.origin = '3004'
            else:
                # I-15 at the Montana/Alberta border
                self.origin = '3310'
        else:
            self.origin = pick_county(MAKE_DICT, self.sctg, self.origin)

    def get_destination(self):
        # Is the truck going to Alaska?
        if self.destination == '20':
{'fraction_non_alphanumeric': 0.09736411932979158, 'fraction_numerical': 0.019207192480588477, 'mean_word_length': 1.8218506774286538, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '782184', 'n_tokens_mistral': 3011, 'n_tokens_neox': 2813, 'n_words': 948}