# Path: recommender/feed_util.py
# def GetEntryDate(entry, min_date=None, max_date=None):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


from datetime import datetime
from datetime import timedelta
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.ext import db
from google.appengine.ext import ndb
from recommender import feed_util
from recommender import items
from recommender import url_util
from protos import cache_pb2
import io
import logging
import time
import feedparser
      else:
        stream = io.BytesIO(content)
      parsed = feedparser.parse(stream)
    else:
      try:
        parsed = feedparser.parse(url, etag=etag, modified=modified_tuple)
      except StandardError as e:
        logging.warning('Failed to parse feed: %s error: %s', url, e)
        # Try to read content ourselves and pass it to the parser:
        try:
          content, _, _, _ = url_util.GetPageContent(url)
          parsed = feedparser.parse(content)
        except url_util.Error as e:
          logging.warning('Failed to parse feed %s with fallback error: %s',
                          url, e)
          raise ndb.Return([])
    if hasattr(parsed, 'modified_parsed') and parsed.modified_parsed:
      self.modified = datetime.fromtimestamp(
          time.mktime(parsed.modified_parsed))
    if hasattr(parsed, 'etag'):
      self.etag = parsed.etag
    new_items = []
    for entry in parsed.entries:
      if not hasattr(entry, 'link') or not entry.link:
        continue
      if hasattr(entry, 'id') and entry.id:
        entry_id = entry.id
      else:
        # Some feeds don't specify the id so we use the link as the id.
        entry_id = entry.link

# Path: recommender/feed_util.py
# def GetEntryDate(entry, min_date=None, max_date=None):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


# Path: recommender/feeds.py
from datetime import datetime
from datetime import timedelta
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.ext import db
from google.appengine.ext import ndb
from recommender import feed_util
from recommender import items
from recommender import url_util
from protos import cache_pb2
import io
import logging
import time
import feedparser

      else:
        stream = io.BytesIO(content)
      parsed = feedparser.parse(stream)
    else:
      try:
        parsed = feedparser.parse(url, etag=etag, modified=modified_tuple)
      except StandardError as e:
        logging.warning('Failed to parse feed: %s error: %s', url, e)
        # Try to read content ourselves and pass it to the parser:
        try:
          content, _, _, _ = url_util.GetPageContent(url)
          parsed = feedparser.parse(content)
        except url_util.Error as e:
          logging.warning('Failed to parse feed %s with fallback error: %s',
                          url, e)
          raise ndb.Return([])
    if hasattr(parsed, 'modified_parsed') and parsed.modified_parsed:
      self.modified = datetime.fromtimestamp(
          time.mktime(parsed.modified_parsed))
    if hasattr(parsed, 'etag'):
      self.etag = parsed.etag
    new_items = []
    for entry in parsed.entries:
      if not hasattr(entry, 'link') or not entry.link:
        continue
      if hasattr(entry, 'id') and entry.id:
        entry_id = entry.id
      else:
        # Some feeds don't specify the id so we use the link as the id.
        entry_id = entry.link

      entry_date = feed_util.GetEntryDate(