# Path: newsApp/feed.py
# class Feed(DbItem):
#   """
#   Represents a web feed.
# 
#   Each feed consists of a unique identifier and a set of tags(key-value pairs).
#   """
# 
# Path: newsApp/feedProcessor.py
# def getLinksFromWebFeed(jobId, feed):
#   feedAndJobId = "Feed id: " + feed.id + ". Job id: " + jobId
# 
#   # get page html
#   pageHtml = ""
#   if FEEDTAG_IS_FEEDPAGE_STATIC in feed.tags:
#     try:
#       pageHtml = getHtmlStatic(feed.tags[FEEDTAG_URL])
#     except (requests.ReadTimeout, requests.ConnectTimeout, requests.ConnectionError):
#       logger.warning(
#         "Timed out or could not connect while getting web feed page %s. Returning zero links. %s",
#         feed.tags[FEEDTAG_URL],
#         feedAndJobId)
#       return []
#   else:
#     pageHtml = loadPageAndGetHtml(feed.tags[FEEDTAG_URL])
#   logger.info("Got html for web page. %s.", feedAndJobId)
# 
#   # load entry selectors
#   entrySelectors = json.loads(feed.tags[FEEDTAG_ENTRY_SELECTORS])
#   logger.info(
#     "Will use %i entry selectors. %s",
#     len(entrySelectors),
#     feedAndJobId)
# 
#   # Use entry selector to get entries
#   links = []
#   for entrySelector in entrySelectors:
#     entries = hp.getSubHtmlEntries(jobId, pageHtml, entrySelector['overall'])
#     logger.info(
#       "Got %i entries for entry selector %s. %s",
#       len(entries),
#       entrySelector['overall'],
#       feedAndJobId)
# 
#     # considering only the top 30 entries to reduce load
#     for entry in entries[:30]:
#       link = _linkFromWebPageEntry(jobId, entry, feed, entrySelector)
#       if link:
#         links.append(link)
# 
#   if len(links) == 0:
#     logger.warning("No links found while processing webPage. %s", feedAndJobId)
#   else:
#     logger.info("Number of links found: %i. %s", len(links), feedAndJobId)
# 
#   return links


import newsApp.constants as ct
import unittest
from newsApp.feed import Feed
from newsApp.feedProcessor import getLinksFromWebFeed

class FeedProcessorTests(unittest.TestCase):

  def testProcessWebFeed(self):
    dcHydFeedTags = {}
    dcHydFeedTags[ct.FEEDTAG_URL] = 'http://www.deccanchronicle.com/location/india/telangana/hyderabad'
    dcHydFeedTags[ct.FEEDTAG_LOCALE] = 'hyd'
    dcHydFeedTags[ct.FEEDTAG_IS_FEEDPAGE_STATIC] = 'true'
    dcHydFeedTags[ct.FEEDTAG_ENTRY_SELECTORS] = '[{"overall": ".storyList .SunChNewListing", ' + \
      '"title": ".col-sm-8 a", "titleText": ".col-sm-8 a h3"}]'
    dcHydFeed = Feed('dcHyd', dcHydFeedTags)


# Path: newsApp/feed.py
# class Feed(DbItem):
#   """
#   Represents a web feed.
# 
#   Each feed consists of a unique identifier and a set of tags(key-value pairs).
#   """
# 
# Path: newsApp/feedProcessor.py
# def getLinksFromWebFeed(jobId, feed):
#   feedAndJobId = "Feed id: " + feed.id + ". Job id: " + jobId
# 
#   # get page html
#   pageHtml = ""
#   if FEEDTAG_IS_FEEDPAGE_STATIC in feed.tags:
#     try:
#       pageHtml = getHtmlStatic(feed.tags[FEEDTAG_URL])
#     except (requests.ReadTimeout, requests.ConnectTimeout, requests.ConnectionError):
#       logger.warning(
#         "Timed out or could not connect while getting web feed page %s. Returning zero links. %s",
#         feed.tags[FEEDTAG_URL],
#         feedAndJobId)
#       return []
#   else:
#     pageHtml = loadPageAndGetHtml(feed.tags[FEEDTAG_URL])
#   logger.info("Got html for web page. %s.", feedAndJobId)
# 
#   # load entry selectors
#   entrySelectors = json.loads(feed.tags[FEEDTAG_ENTRY_SELECTORS])
#   logger.info(
#     "Will use %i entry selectors. %s",
#     len(entrySelectors),
#     feedAndJobId)
# 
#   # Use entry selector to get entries
#   links = []
#   for entrySelector in entrySelectors:
#     entries = hp.getSubHtmlEntries(jobId, pageHtml, entrySelector['overall'])
#     logger.info(
#       "Got %i entries for entry selector %s. %s",
#       len(entries),
#       entrySelector['overall'],
#       feedAndJobId)
# 
#     # considering only the top 30 entries to reduce load
#     for entry in entries[:30]:
#       link = _linkFromWebPageEntry(jobId, entry, feed, entrySelector)
#       if link:
#         links.append(link)
# 
#   if len(links) == 0:
#     logger.warning("No links found while processing webPage. %s", feedAndJobId)
#   else:
#     logger.info("Number of links found: %i. %s", len(links), feedAndJobId)
# 
#   return links


# Path: tests/feedProcessor_test.py
import newsApp.constants as ct
import unittest
from newsApp.feed import Feed
from newsApp.feedProcessor import getLinksFromWebFeed


class FeedProcessorTests(unittest.TestCase):

  def testProcessWebFeed(self):
    dcHydFeedTags = {}
    dcHydFeedTags[ct.FEEDTAG_URL] = 'http://www.deccanchronicle.com/location/india/telangana/hyderabad'
    dcHydFeedTags[ct.FEEDTAG_LOCALE] = 'hyd'
    dcHydFeedTags[ct.FEEDTAG_IS_FEEDPAGE_STATIC] = 'true'
    dcHydFeedTags[ct.FEEDTAG_ENTRY_SELECTORS] = '[{"overall": ".storyList .SunChNewListing", ' + \
      '"title": ".col-sm-8 a", "titleText": ".col-sm-8 a h3"}]'
    dcHydFeed = Feed('dcHyd', dcHydFeedTags)


    links = getLinksFromWebFeed('testJob', dcHydFeed)