<commit_before>
import logging
import pickle
import re
from datetime import datetime, timedelta

from google.appengine.ext import db
from google.appengine.ext import deferred

from models import CHPIncident
from utils.tzinfo import Pacific
from thirdparty.pubsubhubbub_publish import *


def process_chp_xml(chpState):
	for chpCenter in chpState:
		deferred.defer(process_chp_center, chpCenter);


def process_chp_center(chpCenter):
	incident_list = []
	psh_pings = []
	
	for chpDispatch in chpCenter:
		# For some reason, sometimes the Dispatch ID is blank
		# so skip these.
		if chpDispatch.attrib['ID'] == "":
			continue

		for chpLog in chpDispatch:
			incident = CHPIncident.get_by_key_name(chpLog.attrib['ID'])
			if incident is None:
				incident = CHPIncident(key_name = chpLog.attrib['ID'],
					CenterID = chpCenter.attrib['ID'],
					DispatchID = chpDispatch.attrib['ID'],
					LogID = chpLog.attrib['ID'],
					LogTime = datetime.strptime(chpLog.find('LogTime').text, '"%m/%d/%Y %I:%M:%S %p"').replace(tzinfo=Pacific()),<commit_msg>Oh crap, they re-use IncidentIDs.  Pad them out to make 'em unique.<commit_after>
import logging
import pickle
import re
import time
from datetime import datetime, timedelta

from google.appengine.ext import db
from google.appengine.ext import deferred

from models import CHPIncident
from utils.tzinfo import Pacific
from thirdparty.pubsubhubbub_publish import *


def process_chp_xml(chpState):
	for chpCenter in chpState:
		deferred.defer(process_chp_center, chpCenter);


def process_chp_center(chpCenter):
	incident_list = []
	psh_pings = []
	
	for chpDispatch in chpCenter:
		# For some reason, sometimes the Dispatch ID is blank
		# so skip these.
		if chpDispatch.attrib['ID'] == "":
			continue

		for chpLog in chpDispatch:
			log_time = datetime.strptime(chpLog.find('LogTime').text, '"%m/%d/%Y %I:%M:%S %p"').replace(tzinfo=Pacific())
			key_name = "%s.%s.%s.%d" % (chpCenter.attrib['ID'], chpDispatch.attrib['ID'], chpLog.attrib['ID'], time.mktime(log_time.timetuple()))

			incident = CHPIncident.get_by_key_name(key_name)
			if incident is None:
				incident = CHPIncident(key_name = key_name,
					CenterID = chpCenter.attrib['ID'],
					DispatchID = chpDispatch.attrib['ID'],
					LogID = chpLog.attrib['ID'],
					LogTime = log_time,
{'fraction_non_alphanumeric': 0.11710927296473661, 'fraction_numerical': 0.0, 'mean_word_length': 4.773869346733668, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '651742', 'n_tokens_mistral': 888, 'n_tokens_neox': 797, 'n_words': 192}