# Path: recommender/feed_util.py
# def GetEntryDate(entry, min_date=None, max_date=None):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


from datetime import datetime
from datetime import timedelta
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.ext import db
from google.appengine.ext import ndb
from recommender import feed_util
from recommender import items
from recommender import url_util
from protos import cache_pb2
import io
import logging
import time
import feedparser
  etag = ndb.StringProperty()
  last_updated = ndb.DateTimeProperty()

  def GetUrl(self):
    return self.key.id()

  def Update(self, canonicalize=None):
    return self.UpdateAsync(
        canonicalize=canonicalize, enable_async=False).get_result()

  # Updates the list of items in the feed.
  # Returns the list of newly added items.
  @ndb.tasklet
  def UpdateAsync(self, canonicalize=None, enable_async=True):
    now = datetime.now()
    if canonicalize is None:
      canonicalize = lambda url: url
    modified_tuple = None
    etag = self.etag
    if self.modified is not None:
      modified_tuple = self.modified.timetuple()
      # Don't use etag if we have modified time. Some servers are generating new
      # etags when nothing actually changes and modified date stays the same.
      etag = None
    url = self.GetUrl()
    if enable_async:
      if url.startswith('feed:http'):
        url = url[5:]
      elif url.startswith('feed:'):
        url = 'http:' + url[5:]

# Path: recommender/feed_util.py
# def GetEntryDate(entry, min_date=None, max_date=None):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


# Path: recommender/feeds.py
from datetime import datetime
from datetime import timedelta
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.ext import db
from google.appengine.ext import ndb
from recommender import feed_util
from recommender import items
from recommender import url_util
from protos import cache_pb2
import io
import logging
import time
import feedparser

  etag = ndb.StringProperty()
  last_updated = ndb.DateTimeProperty()

  def GetUrl(self):
    return self.key.id()

  def Update(self, canonicalize=None):
    return self.UpdateAsync(
        canonicalize=canonicalize, enable_async=False).get_result()

  # Updates the list of items in the feed.
  # Returns the list of newly added items.
  @ndb.tasklet
  def UpdateAsync(self, canonicalize=None, enable_async=True):
    now = datetime.now()
    if canonicalize is None:
      canonicalize = lambda url: url
    modified_tuple = None
    etag = self.etag
    if self.modified is not None:
      modified_tuple = self.modified.timetuple()
      # Don't use etag if we have modified time. Some servers are generating new
      # etags when nothing actually changes and modified date stays the same.
      etag = None
    url = self.GetUrl()
    if enable_async:
      if url.startswith('feed:http'):
        url = url[5:]
      elif url.startswith('feed:'):
        url = 'http:' + url[5:]

      rpc = urlfetch.create_rpc(deadline=url_util.URL_FETCH_DEADLINE_SECONDS)