<commit_before>#!/usr/bin/env python

import wsgiref.handlers
import logging
import urllib
import datetime
import json
import time

from xml.dom.minidom import getDOMImplementation, parseString

from Location import Location
from MovieListings import MovieListings
from TrailerListings import TrailerListings

from google.appengine.ext import webapp
from google.appengine.ext import db
from google.appengine.api import memcache
from google.appengine.api import urlfetch

class LookupNumbersListingsHandler(webapp.RequestHandler):
  def get(self):
    #memcache.flush_all()    
    q = self.request.get("q")
    id = self.request.get("id")
    hash = self.request.get("hash")

    response = None
    if q == "index":
      response = self.get_index_from_cache(hash)
    else:
      response = self.get_numbers_from_cache(hash, id)

    if response is None:
      response = ""

    self.response.out.write(response)


  def get_index_from_cache(self, h):
    key = "NumbersListings_Index"
    listings = memcache.get(key)

    if listings is None:
      listings = self.get_index_from_site()
      memcache.Client().set(key, listings, 8 * 60 * 60)

    return listings
    

  def get_numbers_from_cache(self, hash, id):
    key = "NumbersListings_" + id
    listings = memcache.get(key)

    if listings is None:
      listings = self.get_numbers_from_site(id)
      memcache.Client().set(key, listings, 8 * 60 * 60)

    return listings


  def get_numbers_from_site(self, name):
    url = "http://access.opusdata.com/session/create?email=<EMAIL>&password=<PASSWORD>"
    content = urlfetch.fetch(url).content

    object = json.read(content)
    
    if not object.has_key("session_id") or not object.has_key("server"):
      return None

    session_id = object["session_id"]
    server = object["server"]
  
    url = "http://" + server + "/movie_theatrical_chart/oe" + name + "/session_id=" + session_id
    time.sleep(1)
    weekend = urlfetch.fetch(url).content

    url = "http://" + server + "/movie_theatrical_chart/oy" + name + "/session_id=" + session_id
    time.sleep(0.5)
    daily = urlfetch.fetch(url).content

    url = "http://" + server + "/movie/" + name + "/session_id=" + session_id
    time.sleep(0.5)
    movie = urlfetch.fetch(url).content

    urlfetch.fetch("http://access.opusdata.com/session/close/" + session_id)

    weekend_json = json.read(weekend)
    daily_json = json.read(daily)
    movie_json = json.read(movie)

    if (not weekend_json.has_key("chart_entries") or
        not daily_json.has_key("chart_entries")):
      return None

    document = getDOMImplementation().createDocument(None, "result", None)
    resultElement = document.documentElement
    if movie_json.has_key("production_cost"):
      resultElement.setAttribute("budget", movie_json["production_cost"])

    weekendElement = document.createElement("weekend")
    dailyElement = document.createElement("daily")
    resultElement.appendChild(weekendElement)
    resultElement.appendChild(dailyElement)

    if len(weekend_json["chart_entries"]) >= 2:
      for entry in weekend_json["chart_entries"][-2:]:
        movie_element = self.convert_to_xml(document, entry)
        weekendElement.appendChild(movie_element)
      
    if len(daily_json["chart_entries"]) >= 2:
      for entry in daily_json["chart_entries"][-2:]:
        movie_element = self.convert_to_xml(document, entry)
        dailyElement.appendChild(movie_element)

    return document.toxml()


  def get_index_from_site(self):
    url = "http://access.opusdata.com/session/create?email=<EMAIL>&password=<PASSWORD>"
    content = urlfetch.fetch(url).content

    object = json.read(content)
    
    if not object.has_key("session_id") or not object.has_key("server"):
      return None

    session_id = object["session_id"]
    server = object["server"]
  
    url = "http://" + server + "/movie_theatrical_chart/we99999999/session_id=" + session_id
    time.sleep(1)
    weekend = urlfetch.fetch(url).content

    url = "http://" + server + "/movie_theatrical_chart/da99999999/session_id=" + session_id
    time.sleep(0.5)
    daily = urlfetch.fetch(url).content

    urlfetch.fetch("http://access.opusdata.com/session/close/" + session_id)

    weekend_json = json.read(weekend)
    daily_json = json.read(daily)

    if not weekend_json.has_key("chart_entries") or not daily_json.has_key("chart_entries"):
      return None

    document = getDOMImplementation().createDocument(None, "result", None)
    resultElement = document.documentElement

    weekendElement = document.createElement("weekend")<commit_msg>getting rid of sleeps.  bruce says i don't need them<commit_after>#!/usr/bin/env python

import wsgiref.handlers
import logging
import urllib
import datetime
import json
import time

from xml.dom.minidom import getDOMImplementation, parseString

from Location import Location
from MovieListings import MovieListings
from TrailerListings import TrailerListings

from google.appengine.ext import webapp
from google.appengine.ext import db
from google.appengine.api import memcache
from google.appengine.api import urlfetch

class LookupNumbersListingsHandler(webapp.RequestHandler):
  def get(self):
    #memcache.flush_all()    
    q = self.request.get("q")
    id = self.request.get("id")
    hash = self.request.get("hash")

    response = None
    if q == "index":
      response = self.get_index_from_cache(hash)
    else:
      response = self.get_numbers_from_cache(hash, id)

    if response is None:
      response = ""

    self.response.out.write(response)


  def get_index_from_cache(self, h):
    key = "NumbersListings_Index"
    listings = memcache.get(key)

    if listings is None:
      listings = self.get_index_from_site()
      memcache.Client().set(key, listings, 8 * 60 * 60)

    return listings
    

  def get_numbers_from_cache(self, hash, id):
    key = "NumbersListings_" + id
    listings = memcache.get(key)

    if listings is None:
      listings = self.get_numbers_from_site(id)
      memcache.Client().set(key, listings, 8 * 60 * 60)

    return listings


  def get_numbers_from_site(self, name):
    url = "http://access.opusdata.com/session/create?email=<EMAIL>&password=<PASSWORD>"
    content = urlfetch.fetch(url).content

    object = json.read(content)
    
    if not object.has_key("session_id") or not object.has_key("server"):
      return None

    session_id = object["session_id"]
    server = object["server"]
  
    url = "http://" + server + "/movie_theatrical_chart/oe" + name + "/session_id=" + session_id
    weekend = urlfetch.fetch(url).content

    url = "http://" + server + "/movie_theatrical_chart/oy" + name + "/session_id=" + session_id
    daily = urlfetch.fetch(url).content

    url = "http://" + server + "/movie/" + name + "/session_id=" + session_id
    movie = urlfetch.fetch(url).content

    urlfetch.fetch("http://access.opusdata.com/session/close/" + session_id)

    weekend_json = json.read(weekend)
    daily_json = json.read(daily)
    movie_json = json.read(movie)

    if (not weekend_json.has_key("chart_entries") or
        not daily_json.has_key("chart_entries")):
      return None

    document = getDOMImplementation().createDocument(None, "result", None)
    resultElement = document.documentElement
    if movie_json.has_key("production_cost"):
      resultElement.setAttribute("budget", movie_json["production_cost"])

    weekendElement = document.createElement("weekend")
    dailyElement = document.createElement("daily")
    resultElement.appendChild(weekendElement)
    resultElement.appendChild(dailyElement)

    if len(weekend_json["chart_entries"]) >= 2:
      for entry in weekend_json["chart_entries"][-2:]:
        movie_element = self.convert_to_xml(document, entry)
        weekendElement.appendChild(movie_element)
      
    if len(daily_json["chart_entries"]) >= 2:
      for entry in daily_json["chart_entries"][-2:]:
        movie_element = self.convert_to_xml(document, entry)
        dailyElement.appendChild(movie_element)

    return document.toxml()


  def get_index_from_site(self):
    url = "http://access.opusdata.com/session/create?email=<EMAIL>&password=<PASSWORD>"
    content = urlfetch.fetch(url).content

    object = json.read(content)
    
    if not object.has_key("session_id") or not object.has_key("server"):
      return None

    session_id = object["session_id"]
    server = object["server"]
  
    url = "http://" + server + "/movie_theatrical_chart/we99999999/session_id=" + session_id
    weekend = urlfetch.fetch(url).content

    url = "http://" + server + "/movie_theatrical_chart/da99999999/session_id=" + session_id
    daily = urlfetch.fetch(url).content

    urlfetch.fetch("http://access.opusdata.com/session/close/" + session_id)

    weekend_json = json.read(weekend)
    daily_json = json.read(daily)

    if not weekend_json.has_key("chart_entries") or not daily_json.has_key("chart_entries"):
      return None

    document = getDOMImplementation().createDocument(None, "result", None)
    resultElement = document.documentElement

    weekendElement = document.createElement("weekend")
{'fraction_non_alphanumeric': 0.11978539362750465, 'fraction_numerical': 0.007445527209022227, 'mean_word_length': 3.75481520041645, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1355870', 'n_tokens_mistral': 2973, 'n_tokens_neox': 2816, 'n_words': 567}