# Path: recommender/feeds.py
# MAX_ITEMS_PER_FEED = 0
# CLEAN_UP_ITEMS_PER_UPDATE = 1000
# TIME_BETWEEN_UPDATES = timedelta(minutes=30)
# CACHE_TIME_PERIODS_IN_DAYS = [1, 3, 7, 14, 30, 60, 100000]
# ITEM_ID_CACHE_PREFIX = 'fid:'
# class Feed(ndb.Model):
# class FeedItemId(object):
# class FeedItem(ndb.Model):
#   def GetUrl(self):
#   def Update(self, canonicalize=None):
#   def UpdateAsync(self, canonicalize=None, enable_async=True):
# def _TimeToMillis(t):
# def _MillisToDatetime(ms):
#   def __init__(self, feed_url, item_id, published_date):
# def GetBulkItemIdsAsync(feed_urls, since_time):
# def _UpdateCachedItemIds(feed_url, time_period_in_days):
# def GetFeed(url):
# def AddFeed(url, title):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/json_encoder.py
# class Serializable(object):
# class JSONEncoder(json.JSONEncoder):
#   def to_dict(self):
# def TimeToMillis(t):
#   def default(self, o):
# 
# Path: recommender/ratings.py
# POSITIVE = 1
# NEGATIVE = -1
# NEUTRAL = 0
# def NumberToRating(value):
# 
# Path: recommender/time_periods.py
# LAST_VISIT_RESTRICTED = 'LAST_VISIT_RESTRICTED'
# LAST_VISIT = 'LAST_VISIT'
# RECENT = 'RECENT'
# HOUR = 'HOUR'
# DAY = 'DAY'
# WEEK = 'WEEK'
# MONTH = 'MONTH'
# YEAR = 'YEAR'
# ALL = 'ALL'
# ALL_NUMERIC = 0
# YEAR_NUMERIC = 1
# MONTH_NUMERIC = 2
# WEEK_NUMERIC = 3
# DAY_NUMERIC = 4
# HOUR_NUMERIC = 5
# RECENT_NUMERIC = 6
# LAST_VISIT_NUMERIC = 7
# LAST_VISIT_RESTRICTED_NUMERIC = 8
# TIME_PERIODS = [
#     {
#         'timedelta': timedelta.max,
#         'name': ALL,
#         'numeric': ALL_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=365),
#         'name': YEAR,
#         'numeric': YEAR_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=31),
#         'name': MONTH,
#         'numeric': MONTH_NUMERIC
#     },
#     {
#         'timedelta': timedelta(weeks=1),
#         'name': WEEK,
#         'numeric': WEEK_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=1),
#         'name': DAY,
#         'numeric': DAY_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=1),
#         'name': HOUR,
#         'numeric': HOUR_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': RECENT,
#         'numeric': RECENT_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': LAST_VISIT,
#         'numeric': LAST_VISIT_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': LAST_VISIT_RESTRICTED,
#         'numeric': LAST_VISIT_RESTRICTED_NUMERIC
#     },
# ]
# def Get(name):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


from datetime import datetime
from datetime import timedelta
from urlparse import urlparse
from google.appengine.api import memcache
from google.appengine.ext import deferred
from google.appengine.ext import ndb
from recommender import feeds
from recommender import items
from recommender import json_encoder
from recommender import ratings
from recommender import time_periods
from recommender import url_util
import functools
import logging
import math
import pickle
  ]

  # Find feed items for this url.
  feed_ratings = [
      FeedRatingToUnified(item) for item in feeds.FeedItem.query(
          feeds.FeedItem.url == url).order(-feeds.FeedItem.published_date)
  ]

  # We do not include the source feed if this url was downvoted because we
  # do not trust that the feed url in the page info has posted this url.
  # This is to prevent the case when a "bad" page tries to lower the reputation
  # of a "good" feed and the "bad" page puts a link to the "good" feed in the
  # <head> <meta> tag.
  if include_source_feed:
    page_info = page_info_future.get_result()
    if page_info.feed_url:
      feed_already_included = False
      for feed_rating in feed_ratings:
        if feed_rating['publisher_id'] == page_info.feed_url:
          feed_already_included = True
          break
      if not feed_already_included:
        feed_ratings.append({
            'type': SOURCE_TYPE_FEED,
            'publisher_id': page_info.feed_url,
            'publisher_category_id': None,
            # The "rating" time is used to determine how much the feed has
            # posted since then. So we set it just one day ago so the feed
            # does not get penalized for the old content.
            'date': datetime.now() - timedelta(days=1),

# Path: recommender/feeds.py
# MAX_ITEMS_PER_FEED = 0
# CLEAN_UP_ITEMS_PER_UPDATE = 1000
# TIME_BETWEEN_UPDATES = timedelta(minutes=30)
# CACHE_TIME_PERIODS_IN_DAYS = [1, 3, 7, 14, 30, 60, 100000]
# ITEM_ID_CACHE_PREFIX = 'fid:'
# class Feed(ndb.Model):
# class FeedItemId(object):
# class FeedItem(ndb.Model):
#   def GetUrl(self):
#   def Update(self, canonicalize=None):
#   def UpdateAsync(self, canonicalize=None, enable_async=True):
# def _TimeToMillis(t):
# def _MillisToDatetime(ms):
#   def __init__(self, feed_url, item_id, published_date):
# def GetBulkItemIdsAsync(feed_urls, since_time):
# def _UpdateCachedItemIds(feed_url, time_period_in_days):
# def GetFeed(url):
# def AddFeed(url, title):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/json_encoder.py
# class Serializable(object):
# class JSONEncoder(json.JSONEncoder):
#   def to_dict(self):
# def TimeToMillis(t):
#   def default(self, o):
# 
# Path: recommender/ratings.py
# POSITIVE = 1
# NEGATIVE = -1
# NEUTRAL = 0
# def NumberToRating(value):
# 
# Path: recommender/time_periods.py
# LAST_VISIT_RESTRICTED = 'LAST_VISIT_RESTRICTED'
# LAST_VISIT = 'LAST_VISIT'
# RECENT = 'RECENT'
# HOUR = 'HOUR'
# DAY = 'DAY'
# WEEK = 'WEEK'
# MONTH = 'MONTH'
# YEAR = 'YEAR'
# ALL = 'ALL'
# ALL_NUMERIC = 0
# YEAR_NUMERIC = 1
# MONTH_NUMERIC = 2
# WEEK_NUMERIC = 3
# DAY_NUMERIC = 4
# HOUR_NUMERIC = 5
# RECENT_NUMERIC = 6
# LAST_VISIT_NUMERIC = 7
# LAST_VISIT_RESTRICTED_NUMERIC = 8
# TIME_PERIODS = [
#     {
#         'timedelta': timedelta.max,
#         'name': ALL,
#         'numeric': ALL_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=365),
#         'name': YEAR,
#         'numeric': YEAR_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=31),
#         'name': MONTH,
#         'numeric': MONTH_NUMERIC
#     },
#     {
#         'timedelta': timedelta(weeks=1),
#         'name': WEEK,
#         'numeric': WEEK_NUMERIC
#     },
#     {
#         'timedelta': timedelta(days=1),
#         'name': DAY,
#         'numeric': DAY_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=1),
#         'name': HOUR,
#         'numeric': HOUR_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': RECENT,
#         'numeric': RECENT_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': LAST_VISIT,
#         'numeric': LAST_VISIT_NUMERIC
#     },
#     {
#         'timedelta': timedelta(hours=0),
#         'name': LAST_VISIT_RESTRICTED,
#         'numeric': LAST_VISIT_RESTRICTED_NUMERIC
#     },
# ]
# def Get(name):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


# Path: recommender/models.py
from datetime import datetime
from datetime import timedelta
from urlparse import urlparse
from google.appengine.api import memcache
from google.appengine.ext import deferred
from google.appengine.ext import ndb
from recommender import feeds
from recommender import items
from recommender import json_encoder
from recommender import ratings
from recommender import time_periods
from recommender import url_util
import functools
import logging
import math
import pickle

  ]

  # Find feed items for this url.
  feed_ratings = [
      FeedRatingToUnified(item) for item in feeds.FeedItem.query(
          feeds.FeedItem.url == url).order(-feeds.FeedItem.published_date)
  ]

  # We do not include the source feed if this url was downvoted because we
  # do not trust that the feed url in the page info has posted this url.
  # This is to prevent the case when a "bad" page tries to lower the reputation
  # of a "good" feed and the "bad" page puts a link to the "good" feed in the
  # <head> <meta> tag.
  if include_source_feed:
    page_info = page_info_future.get_result()
    if page_info.feed_url:
      feed_already_included = False
      for feed_rating in feed_ratings:
        if feed_rating['publisher_id'] == page_info.feed_url:
          feed_already_included = True
          break
      if not feed_already_included:
        feed_ratings.append({
            'type': SOURCE_TYPE_FEED,
            'publisher_id': page_info.feed_url,
            'publisher_category_id': None,
            # The "rating" time is used to determine how much the feed has
            # posted since then. So we set it just one day ago so the feed
            # does not get penalized for the old content.
            'date': datetime.now() - timedelta(days=1),

            'rating': ratings.POSITIVE,