# Path: recommender/feed_util.py
# def GetEntryDate(entry, min_date=None, max_date=None):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


from datetime import datetime
from datetime import timedelta
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.ext import db
from google.appengine.ext import ndb
from recommender import feed_util
from recommender import items
from recommender import url_util
from protos import cache_pb2
import io
import logging
import time
import feedparser
    if hasattr(parsed, 'etag'):
      self.etag = parsed.etag
    new_items = []
    for entry in parsed.entries:
      if not hasattr(entry, 'link') or not entry.link:
        continue
      if hasattr(entry, 'id') and entry.id:
        entry_id = entry.id
      else:
        # Some feeds don't specify the id so we use the link as the id.
        entry_id = entry.link
      entry_date = feed_util.GetEntryDate(
          entry, min_date=self.last_updated, max_date=now)
      if not entry_date:
        entry_date = datetime.now()
      title = None
      if hasattr(entry, 'title'):
        title = entry.title
      item = FeedItem.query(FeedItem.feed_url == self.GetUrl(),
                            FeedItem.id == entry_id).get()
      if item is None:
        try:
          canonical_url = canonicalize(entry.link)
          new_item = FeedItem(
              id=entry_id,
              url=canonical_url,
              title=title,
              feed_url=self.GetUrl(),
              retrieved_date=now,
              published_date=entry_date,

# Path: recommender/feed_util.py
# def GetEntryDate(entry, min_date=None, max_date=None):
# 
# Path: recommender/items.py
# class Item(ndb.Model):
#   def ItemId(self):
# def UrlToItemId(url):
# def UrlsToItemIds(urls):
# def ItemIdsToUrls(item_ids):
# def ItemIdToUrl(item_id):
# def ItemIdsToBytes(item_ids):
# def ItemIdsFromBytes(string):
# 
# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


# Path: recommender/feeds.py
from datetime import datetime
from datetime import timedelta
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.ext import db
from google.appengine.ext import ndb
from recommender import feed_util
from recommender import items
from recommender import url_util
from protos import cache_pb2
import io
import logging
import time
import feedparser

    if hasattr(parsed, 'etag'):
      self.etag = parsed.etag
    new_items = []
    for entry in parsed.entries:
      if not hasattr(entry, 'link') or not entry.link:
        continue
      if hasattr(entry, 'id') and entry.id:
        entry_id = entry.id
      else:
        # Some feeds don't specify the id so we use the link as the id.
        entry_id = entry.link
      entry_date = feed_util.GetEntryDate(
          entry, min_date=self.last_updated, max_date=now)
      if not entry_date:
        entry_date = datetime.now()
      title = None
      if hasattr(entry, 'title'):
        title = entry.title
      item = FeedItem.query(FeedItem.feed_url == self.GetUrl(),
                            FeedItem.id == entry_id).get()
      if item is None:
        try:
          canonical_url = canonicalize(entry.link)
          new_item = FeedItem(
              id=entry_id,
              url=canonical_url,
              title=title,
              feed_url=self.GetUrl(),
              retrieved_date=now,
              published_date=entry_date,

              item_id=items.UrlToItemId(canonical_url))