# Path: recommender/feeds.py
# MAX_ITEMS_PER_FEED = 0
# CLEAN_UP_ITEMS_PER_UPDATE = 1000
# TIME_BETWEEN_UPDATES = timedelta(minutes=30)
# CACHE_TIME_PERIODS_IN_DAYS = [1, 3, 7, 14, 30, 60, 100000]
# ITEM_ID_CACHE_PREFIX = 'fid:'
# class Feed(ndb.Model):
# class FeedItemId(object):
# class FeedItem(ndb.Model):
#   def GetUrl(self):
#   def Update(self, canonicalize=None):
#   def UpdateAsync(self, canonicalize=None, enable_async=True):
# def _TimeToMillis(t):
# def _MillisToDatetime(ms):
#   def __init__(self, feed_url, item_id, published_date):
# def GetBulkItemIdsAsync(feed_urls, since_time):
# def _UpdateCachedItemIds(feed_url, time_period_in_days):
# def GetFeed(url):
# def AddFeed(url, title):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/json_encoder.py
# class Serializable(object):
# class JSONEncoder(json.JSONEncoder):
#   def to_dict(self):
# def TimeToMillis(t):
#   def default(self, o):
# 
# Path: recommender/ratings.py
# POSITIVE = 1
# NEGATIVE = -1
# NEUTRAL = 0
# def NumberToRating(value):
# 
# Path: recommender/time_periods.py
# LAST_VISIT_RESTRICTED = 'LAST_VISIT_RESTRICTED'
# LAST_VISIT = 'LAST_VISIT'
# RECENT = 'RECENT'
# HOUR = 'HOUR'
# DAY = 'DAY'
# WEEK = 'WEEK'
# MONTH = 'MONTH'
# YEAR = 'YEAR'
# ALL = 'ALL'
# ALL_NUMERIC = 0
# YEAR_NUMERIC = 1
# MONTH_NUMERIC = 2
# WEEK_NUMERIC = 3
# DAY_NUMERIC = 4
# HOUR_NUMERIC = 5
# RECENT_NUMERIC = 6
# LAST_VISIT_NUMERIC = 7
# LAST_VISIT_RESTRICTED_NUMERIC = 8
# TIME_PERIODS = [
#     {
#         'timedelta': timedelta.max,
#         'name': ALL,
#         'numeric': ALL_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=365),
#         'name': YEAR,
#         'numeric': YEAR_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=31),
#         'name': MONTH,
#         'numeric': MONTH_NUMERIC
#     },
#     {
#         'timedelta': timedelta(weeks=1),
#         'name': WEEK,
#         'numeric': WEEK_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=1),
#         'name': DAY,
#         'numeric': DAY_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=1),
#         'name': HOUR,
#         'numeric': HOUR_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': RECENT,
#         'numeric': RECENT_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': LAST_VISIT,
#         'numeric': LAST_VISIT_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': LAST_VISIT_RESTRICTED,
#         'numeric': LAST_VISIT_RESTRICTED_NUMERIC
#     },
# ]
# def Get(name):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


from datetime import datetime
from datetime import timedelta
from urlparse import urlparse
from google.appengine.api import memcache
from google.appengine.ext import deferred
from google.appengine.ext import ndb
from recommender import feeds
from recommender import items
from recommender import json_encoder
from recommender import ratings
from recommender import time_periods
from recommender import url_util
import functools
import logging
import math
import pickle
  if len(url) >= 500:
    raise ndb.Return(None)
  key = ndb.Key(PageInfo, url)
  page_info = None
  if get_from_datastore:
    page_info = yield key.get_async()
  if page_info is None:
    try:
      if log_not_found:
        logging.warning('url not found in cache and datastore: %s', url)
      if do_not_fetch:
        logging.warning('returning empty details: %s', url)
        raise ndb.Return(None)
      metadata = url_util.GetPageMetadata(url)
      page_info = _MetadataToPageInfo(metadata, key, url)
      yield page_info.put_async()
      # If the original link is a redirect
      # (e.g., bit.ly/abc -> example.com/page.html) then the fetched page will
      # be the content of metadata.final_url and if it also matches the
      # canonical then we will be interested in it under the final url.
      # So we save the page info here.
      if (metadata.final_url != url and
          metadata.final_url == metadata.canonical_url):
        yield _MetadataToPageInfo(metadata, ndb.Key(PageInfo,
                                                    metadata.final_url),
                                  metadata.final_url).put_async()
    except url_util.InvalidURLError as e:
      if raise_invalid_url or raise_error:
        raise e
      raise ndb.Return(PageInfo(key=key, url=url, title=url, canonical_url=url))

# Path: recommender/feeds.py
# MAX_ITEMS_PER_FEED = 0
# CLEAN_UP_ITEMS_PER_UPDATE = 1000
# TIME_BETWEEN_UPDATES = timedelta(minutes=30)
# CACHE_TIME_PERIODS_IN_DAYS = [1, 3, 7, 14, 30, 60, 100000]
# ITEM_ID_CACHE_PREFIX = 'fid:'
# class Feed(ndb.Model):
# class FeedItemId(object):
# class FeedItem(ndb.Model):
#   def GetUrl(self):
#   def Update(self, canonicalize=None):
#   def UpdateAsync(self, canonicalize=None, enable_async=True):
# def _TimeToMillis(t):
# def _MillisToDatetime(ms):
#   def __init__(self, feed_url, item_id, published_date):
# def GetBulkItemIdsAsync(feed_urls, since_time):
# def _UpdateCachedItemIds(feed_url, time_period_in_days):
# def GetFeed(url):
# def AddFeed(url, title):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/json_encoder.py
# class Serializable(object):
# class JSONEncoder(json.JSONEncoder):
#   def to_dict(self):
# def TimeToMillis(t):
#   def default(self, o):
# 
# Path: recommender/ratings.py
# POSITIVE = 1
# NEGATIVE = -1
# NEUTRAL = 0
# def NumberToRating(value):
# 
# Path: recommender/time_periods.py
# LAST_VISIT_RESTRICTED = 'LAST_VISIT_RESTRICTED'
# LAST_VISIT = 'LAST_VISIT'
# RECENT = 'RECENT'
# HOUR = 'HOUR'
# DAY = 'DAY'
# WEEK = 'WEEK'
# MONTH = 'MONTH'
# YEAR = 'YEAR'
# ALL = 'ALL'
# ALL_NUMERIC = 0
# YEAR_NUMERIC = 1
# MONTH_NUMERIC = 2
# WEEK_NUMERIC = 3
# DAY_NUMERIC = 4
# HOUR_NUMERIC = 5
# RECENT_NUMERIC = 6
# LAST_VISIT_NUMERIC = 7
# LAST_VISIT_RESTRICTED_NUMERIC = 8
# TIME_PERIODS = [
#     {
#         'timedelta': timedelta.max,
#         'name': ALL,
#         'numeric': ALL_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=365),
#         'name': YEAR,
#         'numeric': YEAR_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=31),
#         'name': MONTH,
#         'numeric': MONTH_NUMERIC
#     },
#     {
#         'timedelta': timedelta(weeks=1),
#         'name': WEEK,
#         'numeric': WEEK_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=1),
#         'name': DAY,
#         'numeric': DAY_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=1),
#         'name': HOUR,
#         'numeric': HOUR_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': RECENT,
#         'numeric': RECENT_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': LAST_VISIT,
#         'numeric': LAST_VISIT_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': LAST_VISIT_RESTRICTED,
#         'numeric': LAST_VISIT_RESTRICTED_NUMERIC
#     },
# ]
# def Get(name):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


# Path: recommender/models.py
from datetime import datetime
from datetime import timedelta
from urlparse import urlparse
from google.appengine.api import memcache
from google.appengine.ext import deferred
from google.appengine.ext import ndb
from recommender import feeds
from recommender import items
from recommender import json_encoder
from recommender import ratings
from recommender import time_periods
from recommender import url_util
import functools
import logging
import math
import pickle

  if len(url) >= 500:
    raise ndb.Return(None)
  key = ndb.Key(PageInfo, url)
  page_info = None
  if get_from_datastore:
    page_info = yield key.get_async()
  if page_info is None:
    try:
      if log_not_found:
        logging.warning('url not found in cache and datastore: %s', url)
      if do_not_fetch:
        logging.warning('returning empty details: %s', url)
        raise ndb.Return(None)
      metadata = url_util.GetPageMetadata(url)
      page_info = _MetadataToPageInfo(metadata, key, url)
      yield page_info.put_async()
      # If the original link is a redirect
      # (e.g., bit.ly/abc -> example.com/page.html) then the fetched page will
      # be the content of metadata.final_url and if it also matches the
      # canonical then we will be interested in it under the final url.
      # So we save the page info here.
      if (metadata.final_url != url and
          metadata.final_url == metadata.canonical_url):
        yield _MetadataToPageInfo(metadata, ndb.Key(PageInfo,
                                                    metadata.final_url),
                                  metadata.final_url).put_async()
    except url_util.InvalidURLError as e:
      if raise_invalid_url or raise_error:
        raise e
      raise ndb.Return(PageInfo(key=key, url=url, title=url, canonical_url=url))

    except url_util.Error as e: