# Path: recommender/feed_util.py
# def GetEntryDate(entry, min_date=None, max_date=None):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


from datetime import datetime
from datetime import timedelta
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.ext import db
from google.appengine.ext import ndb
from recommender import feed_util
from recommender import items
from recommender import url_util
from protos import cache_pb2
import io
import logging
import time
import feedparser
                                                  etag, modified_tuple,
                                                  referrer, auth,
                                                  request_headers)
      # pylint: enable=protected-access
      headers = {key: value for (key, value) in request.header_items()}
      # We remove this header so that we do not get gzipped content from
      # urlfetch.
      # Without this header passed explicitly urlfetch adds this header
      # implicitly and unzips the response for us.
      headers.pop('Accept-encoding')
      try:
        result = (yield urlfetch.make_fetch_call(rpc, url, headers=headers))
      except urlfetch_errors.Error as e:
        logging.warning('Error fetching feed: %s error: %s', url, e)
        raise ndb.Return([])
      content = result.content
      if isinstance(content, unicode):
        stream = io.BytesIO(content.encode('utf-8'))
      else:
        stream = io.BytesIO(content)
      parsed = feedparser.parse(stream)
    else:
      try:
        parsed = feedparser.parse(url, etag=etag, modified=modified_tuple)
      except StandardError as e:
        logging.warning('Failed to parse feed: %s error: %s', url, e)
        # Try to read content ourselves and pass it to the parser:
        try:
          content, _, _, _ = url_util.GetPageContent(url)
          parsed = feedparser.parse(content)

# Path: recommender/feed_util.py
# def GetEntryDate(entry, min_date=None, max_date=None):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


# Path: recommender/feeds.py
from datetime import datetime
from datetime import timedelta
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.ext import db
from google.appengine.ext import ndb
from recommender import feed_util
from recommender import items
from recommender import url_util
from protos import cache_pb2
import io
import logging
import time
import feedparser

                                                  etag, modified_tuple,
                                                  referrer, auth,
                                                  request_headers)
      # pylint: enable=protected-access
      headers = {key: value for (key, value) in request.header_items()}
      # We remove this header so that we do not get gzipped content from
      # urlfetch.
      # Without this header passed explicitly urlfetch adds this header
      # implicitly and unzips the response for us.
      headers.pop('Accept-encoding')
      try:
        result = (yield urlfetch.make_fetch_call(rpc, url, headers=headers))
      except urlfetch_errors.Error as e:
        logging.warning('Error fetching feed: %s error: %s', url, e)
        raise ndb.Return([])
      content = result.content
      if isinstance(content, unicode):
        stream = io.BytesIO(content.encode('utf-8'))
      else:
        stream = io.BytesIO(content)
      parsed = feedparser.parse(stream)
    else:
      try:
        parsed = feedparser.parse(url, etag=etag, modified=modified_tuple)
      except StandardError as e:
        logging.warning('Failed to parse feed: %s error: %s', url, e)
        # Try to read content ourselves and pass it to the parser:
        try:
          content, _, _, _ = url_util.GetPageContent(url)
          parsed = feedparser.parse(content)

        except url_util.Error as e: