<commit_before>import logging

from google.appengine.api import urlfetch
from google.appengine.api import memcache
from google.appengine.api import quota
from google.appengine.api.urlfetch import DownloadError

from google.appengine.ext import db

from BeautifulSoup import BeautifulSoup, Tag
from data_model import RouteListing
from data_model import BusStopAggregation

from api.v1 import utils

aggregated_results = {}


def clean(sid):
    del aggregated_results[sid]
## end

def cleanAll():
    logging.debug('cleaning the aggregated results list. total of %s sids' % str(len(aggregated_results)))
    for key,value in aggregated_results.items():
        del aggregated_results[key]
    logging.debug('done with the clean... %s' % aggregated_results)
## end

def aggregateBusesAsynch(sid, stopID, routeID=None):
    if len(stopID) == 3:
        stopID = "0" + stopID
        
    routes = getRouteListing(stopID,routeID)
    if len(routes) == 0:
        # this can happen if the user passes in a bogus stopID
        logging.error("API: User error. There are no matching stops for this ID?!? %s" % stopID)
        return None
    else:
    	aggregated_results[sid] = []
        # create a bunch of asynchronous url fetches to get all of the route data
        rpcs = []
        memcache.set(sid,0)
        for r in routes:
            rpc = urlfetch.create_rpc()
            rpc.callback = create_callback(rpc,stopID,r.route,sid,r.direction)
            counter = memcache.incr(sid)
            urlfetch.make_fetch_call(rpc, r.scheduleURL)
            rpcs.append(rpc)
            
        # all of the schedule URLs have been fetched. now wait for them to finish
        for rpc in rpcs:
            rpc.wait()
        
        # all calls should be complete at this point    
        while memcache.get(sid) > 0 :
            logging.info('API: ERROR : uh-oh. in waiting loop... %s' % memcache.get(sid))
            rpc.wait()
            
        return aggregateAsynchResults(sid)
    

## end aggregateBusesAsynch()

#
# once all of the results have been grabbed, piece them together
#
def aggregateAsynchResults(sid):
      logging.info("API: Time to report back on results for %s..." % sid)
      
      if len(aggregated_results[sid]) == 0:
          logging.error("We couldn't find results for transaction %s. Chances are there aren't any matches with the request." % sid)
          textBody = "Doesn't look good... Your bus isn't running right now!"

      return aggregated_results[sid]
  
## end aggregatAsynchResults()

#
# This function handles the callback of a single fetch request.
# If all requests for this sid are services, aggregate the results
#
def handle_result(rpc,stopID,routeID,sid,directionID):
    routes = None
    result = None
    try:
        # go fetch the webpage for this route/stop!
        result = rpc.get_result()
        done = True;
    except urlfetch.DownloadError:
         logging.error("API: Error loading page. route %s, stop %s" % (routeID,stopID))
         if result:
            logging.error("API: Error status: %s" % result.status_code)
            logging.error("API: Error header: %s" % result.headers)
            logging.error("API: Error content: %s" % result.content)
           
    directionLabel = utils.getDirectionLabel(directionID)       
    arrival = '0'
    textBody = 'unknown'
    valid = False
    if result is None or result.status_code != 200:
           logging.error("API: Exiting early: error fetching URL: ")
           textBody = "error " + routeID + " (missing data)"
    else:
           soup = BeautifulSoup(result.content)
           for slot in soup.html.body.findAll("a","ada"):
              # only take the first time entry
              if slot['title'].split(':')[0].isdigit():
                arrival = slot['title']
                textBody = arrival.replace('P.M.','pm').replace('A.M.','am')
                valid = True
                
                # the original implementaiton leveraged the datastore to store and
                # ultimately sort the results when we got all of the routes back.
                # we'll continute to use the model definition, but never actually store
                # the results in the datastore.
                stop = BusStopAggregation()
                stop.stopID = stopID
                stop.routeID = routeID
                stop.sid = sid
                stop.arrivalTime = textBody
                stop.destination = directionLabel
          
                # turn the arrival time into absolute minutes
                hours = int(arrival.split(':')[0])
                if arrival.find('P.M.') > 0 and int(hours) < 12:
                    hours += 12
                minutes = int(arrival.split(':')[1].split()[0])
                arrivalMinutes = (hours * 60) + minutes
                stop.time = arrivalMinutes
          
                stop.text = textBody + " toward %s" % directionLabel
                
                # instead of shoving this in the datastore, we're going to shove
                # it in a local variable and retrieve it with the sid later
                # old implementation --> stop.put()
                insert_result(sid,stop)

    # create the task that glues all the messages together when 
    # we've finished the fetch tasks
    counter = memcache.decr(sid)
    if counter == 0:
        # put them all together
        memcache.delete(sid)
        #routes = aggregateAsynchResults(sid)
        
    return routes

## end

# insert a BusAggregation result into the results array for this SID
# we'll scan the current list and insert it into the array based
# on the time-to-arrival
#
def insert_result(sid,stop):
    if len(aggregated_results[sid]) == 0:
        aggregated_results[sid] = [stop]
    else:
        done = False
        for i, s in enumerate(aggregated_results[sid]):
            if stop.time <= s.time:
                    aggregated_results[sid].insert(i,stop)
                    done = True
                    break
        if not done:
            aggregated_results[sid].append(stop)
            
## end

# Use a helper function to define the scope of the callback.
def create_callback(rpc,stopID,routeID,sid,directionID):
    return lambda: handle_result(rpc,stopID,routeID,sid,directionID)
    
# Convenience method to extract RouteListing
def getRouteListing(stopID,routeID=None):
    
    if routeID is None:
        key = 'routelisting:%s' % stopID
    else:
        key = 'routelisting:%s:%s' % (stopID,routeID)
    
    entities = utils.deserialize_entities(memcache.get(key))
    if not entities:
        if routeID is None:
            entities = db.GqlQuery("SELECT * FROM RouteListing WHERE stopID = :1",stopID).fetch(50)
        else:
            entities = db.GqlQuery("SELECT * FROM RouteListing WHERE stopID = :1 AND route = :2",stopID,routeID).fetch(50)
        if entities:
            memcache.set(key, utils.serialize_entities(entities))
        
    return entities

## end

<commit_msg>reduced the logging level of a message when aggregated results return zero results, which is not uncommon<commit_after>import logging

from google.appengine.api import urlfetch
from google.appengine.api import memcache
from google.appengine.api import quota
from google.appengine.api.urlfetch import DownloadError

from google.appengine.ext import db

from BeautifulSoup import BeautifulSoup, Tag
from data_model import RouteListing
from data_model import BusStopAggregation

from api.v1 import utils

aggregated_results = {}


def clean(sid):
    del aggregated_results[sid]
## end

def cleanAll():
    logging.debug('cleaning the aggregated results list. total of %s sids' % str(len(aggregated_results)))
    for key,value in aggregated_results.items():
        del aggregated_results[key]
    logging.debug('done with the clean... %s' % aggregated_results)
## end

def aggregateBusesAsynch(sid, stopID, routeID=None):
    if len(stopID) == 3:
        stopID = "0" + stopID
        
    routes = getRouteListing(stopID,routeID)
    if len(routes) == 0:
        # this can happen if the user passes in a bogus stopID
        logging.error("API: User error. There are no matching stops for this ID?!? %s" % stopID)
        return None
    else:
    	aggregated_results[sid] = []
        # create a bunch of asynchronous url fetches to get all of the route data
        rpcs = []
        memcache.set(sid,0)
        for r in routes:
            rpc = urlfetch.create_rpc()
            rpc.callback = create_callback(rpc,stopID,r.route,sid,r.direction)
            counter = memcache.incr(sid)
            urlfetch.make_fetch_call(rpc, r.scheduleURL)
            rpcs.append(rpc)
            
        # all of the schedule URLs have been fetched. now wait for them to finish
        for rpc in rpcs:
            rpc.wait()
        
        # all calls should be complete at this point    
        while memcache.get(sid) > 0 :
            logging.info('API: ERROR : uh-oh. in waiting loop... %s' % memcache.get(sid))
            rpc.wait()
            
        return aggregateAsynchResults(sid)
    

## end aggregateBusesAsynch()

#
# once all of the results have been grabbed, piece them together
#
def aggregateAsynchResults(sid):
      logging.info("API: Time to report back on results for %s..." % sid)
      
      if len(aggregated_results[sid]) == 0:
          logging.debug("We couldn't find results for transaction %s. Chances are there aren't any matches with the request." % sid)
          textBody = "Doesn't look good... Your bus isn't running right now!"

      return aggregated_results[sid]
  
## end aggregatAsynchResults()

#
# This function handles the callback of a single fetch request.
# If all requests for this sid are services, aggregate the results
#
def handle_result(rpc,stopID,routeID,sid,directionID):
    routes = None
    result = None
    try:
        # go fetch the webpage for this route/stop!
        result = rpc.get_result()
        done = True;
    except urlfetch.DownloadError:
         logging.error("API: Error loading page. route %s, stop %s" % (routeID,stopID))
         if result:
            logging.error("API: Error status: %s" % result.status_code)
            logging.error("API: Error header: %s" % result.headers)
            logging.error("API: Error content: %s" % result.content)
           
    directionLabel = utils.getDirectionLabel(directionID)       
    arrival = '0'
    textBody = 'unknown'
    valid = False
    if result is None or result.status_code != 200:
           logging.error("API: Exiting early: error fetching URL: ")
           textBody = "error " + routeID + " (missing data)"
    else:
           soup = BeautifulSoup(result.content)
           for slot in soup.html.body.findAll("a","ada"):
              # only take the first time entry
              if slot['title'].split(':')[0].isdigit():
                arrival = slot['title']
                textBody = arrival.replace('P.M.','pm').replace('A.M.','am')
                valid = True
                
                # the original implementaiton leveraged the datastore to store and
                # ultimately sort the results when we got all of the routes back.
                # we'll continute to use the model definition, but never actually store
                # the results in the datastore.
                stop = BusStopAggregation()
                stop.stopID = stopID
                stop.routeID = routeID
                stop.sid = sid
                stop.arrivalTime = textBody
                stop.destination = directionLabel
          
                # turn the arrival time into absolute minutes
                hours = int(arrival.split(':')[0])
                if arrival.find('P.M.') > 0 and int(hours) < 12:
                    hours += 12
                minutes = int(arrival.split(':')[1].split()[0])
                arrivalMinutes = (hours * 60) + minutes
                stop.time = arrivalMinutes
          
                stop.text = textBody + " toward %s" % directionLabel
                
                # instead of shoving this in the datastore, we're going to shove
                # it in a local variable and retrieve it with the sid later
                # old implementation --> stop.put()
                insert_result(sid,stop)

    # create the task that glues all the messages together when 
    # we've finished the fetch tasks
    counter = memcache.decr(sid)
    if counter == 0:
        # put them all together
        memcache.delete(sid)
        #routes = aggregateAsynchResults(sid)
        
    return routes

## end

# insert a BusAggregation result into the results array for this SID
# we'll scan the current list and insert it into the array based
# on the time-to-arrival
#
def insert_result(sid,stop):
    if len(aggregated_results[sid]) == 0:
        aggregated_results[sid] = [stop]
    else:
        done = False
        for i, s in enumerate(aggregated_results[sid]):
            if stop.time <= s.time:
                    aggregated_results[sid].insert(i,stop)
                    done = True
                    break
        if not done:
            aggregated_results[sid].append(stop)
            
## end

# Use a helper function to define the scope of the callback.
def create_callback(rpc,stopID,routeID,sid,directionID):
    return lambda: handle_result(rpc,stopID,routeID,sid,directionID)
    
# Convenience method to extract RouteListing
def getRouteListing(stopID,routeID=None):
    
    if routeID is None:
        key = 'routelisting:%s' % stopID
    else:
        key = 'routelisting:%s:%s' % (stopID,routeID)
    
    entities = utils.deserialize_entities(memcache.get(key))
    if not entities:
        if routeID is None:
            entities = db.GqlQuery("SELECT * FROM RouteListing WHERE stopID = :1",stopID).fetch(50)
        else:
            entities = db.GqlQuery("SELECT * FROM RouteListing WHERE stopID = :1 AND route = :2",stopID,routeID).fetch(50)
        if entities:
            memcache.set(key, utils.serialize_entities(entities))
        
    return entities

## end


{'fraction_non_alphanumeric': 0.0911214953271028, 'fraction_numerical': 0.004389691305579156, 'mean_word_length': 2.1978718587276433, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3772145', 'n_tokens_mistral': 4039, 'n_tokens_neox': 3890, 'n_words': 1242}