# Path: app/aml_utils.py
# def getpage(url):
#     """
#     Downloads the html page
# 
#     :rtype: tuple
#     :param url: the page address
#     :return: the header response and contents (bytes) of the page
#     """
#     http = httplib2.Http('.cache', disable_ssl_certificate_validation=True)
#     headers = {
#         'User-agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36'}
# 
#     response, content = http.request(url, headers=headers)
#     return response, content
# 
# def parsepage(content):
#     """
#     Parses a single page and its contents into a BeautifulSoup object
# 
#     :param content: bytearray
#     :return soup: object
#     """
#     soup = BeautifulSoup(content, 'lxml')
#     return soup
# 
# def get_ns(key):
#     """
#     Gets a news_source filtered by key
#     :param key: the key of the news source (e.g: g1, uol, etc.)
#     :return: the news source object
#     """
#     ns = NewsSource.query.filter_by(
#         key=key).first()  # Fetch news source by key
#     return ns
# 
# Path: app/news_source_national.py
# def g1(soup):
#     """
#     Gets the most read news from the g1 page
# 
#     :param soup: the BeautifulSoup object
#     :return: a list with the most read news from the G1 Page
#     """
#     news = []
#     scripts = soup.find_all('script')
# 
#     for script in scripts:
#         script_content = script.text
# 
#         # O conteúdo do G1 agora é gerado por script. Primeiro achamos o script correto, pois são vários
#         if script_content.find('#G1-POST-TOP') != -1:
#             i = 0
# 
#             # Recuperamos as URLs mais acessadas
#             while True:
#                 # Primeiro achamos um top-post (url) com essa chave de busca
#                 key_index = script_content.find('#G1-POST-TOP', i)
# 
#                 if key_index == -1:
#                     break
# 
#                 # Agora achamos o começo da url
#                 start_index = script_content.rfind('"', 0, key_index) + 1
# 
#                 # Agora achamos o final da url
#                 end_index = script_content.find('"', key_index)
# 
#                 # E agora pegamos a URL (substring)
#                 url = script_content[start_index: end_index]
# 
#                 # Com a URL, entramos na página e descobrimos o título dela
#                 response, content = getpage(url)
#                 soup2 = parsepage(content)
#                 title = soup2.find('h1', class_='content-head__title').string
# 
#                 news.append(dict(title=title, link=url))
# 
#                 # Preparamos o próximo índice de busca
#                 i = key_index + 10
# 
#     return news


from app.aml_utils import getpage, parsepage, get_ns
from app.news_source_national import g1
    return news


def __tec_conv(soup):
    """
    Gets the most read news from the Convergencia Digital page
    :param soup: the BeautifulSoup object
    :return: a list with the most read news from the Convergencia Digital page
    """
    news = []
    ns = get_ns('tec_conv')

    anchors = soup.find('div', id='slideshow').find_all('a')

    for a in anchors:
        if a.h1:
            title = a.h1.string
            link = 'http://www.convergenciadigital.com.br' + a['href']
            news.append(dict(title=title, link=link))
    return news


def __tec_canal(soup):
    """
    DEPRECATED
    Gets the most read news from the Canaltech page
    :param soup: the BeautifulSoup object
    :return: a list with the most read news from the Canaltech page
    """
    news = []

# Path: app/aml_utils.py
# def getpage(url):
#     """
#     Downloads the html page
# 
#     :rtype: tuple
#     :param url: the page address
#     :return: the header response and contents (bytes) of the page
#     """
#     http = httplib2.Http('.cache', disable_ssl_certificate_validation=True)
#     headers = {
#         'User-agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36'}
# 
#     response, content = http.request(url, headers=headers)
#     return response, content
# 
# def parsepage(content):
#     """
#     Parses a single page and its contents into a BeautifulSoup object
# 
#     :param content: bytearray
#     :return soup: object
#     """
#     soup = BeautifulSoup(content, 'lxml')
#     return soup
# 
# def get_ns(key):
#     """
#     Gets a news_source filtered by key
#     :param key: the key of the news source (e.g: g1, uol, etc.)
#     :return: the news source object
#     """
#     ns = NewsSource.query.filter_by(
#         key=key).first()  # Fetch news source by key
#     return ns
# 
# Path: app/news_source_national.py
# def g1(soup):
#     """
#     Gets the most read news from the g1 page
# 
#     :param soup: the BeautifulSoup object
#     :return: a list with the most read news from the G1 Page
#     """
#     news = []
#     scripts = soup.find_all('script')
# 
#     for script in scripts:
#         script_content = script.text
# 
#         # O conteúdo do G1 agora é gerado por script. Primeiro achamos o script correto, pois são vários
#         if script_content.find('#G1-POST-TOP') != -1:
#             i = 0
# 
#             # Recuperamos as URLs mais acessadas
#             while True:
#                 # Primeiro achamos um top-post (url) com essa chave de busca
#                 key_index = script_content.find('#G1-POST-TOP', i)
# 
#                 if key_index == -1:
#                     break
# 
#                 # Agora achamos o começo da url
#                 start_index = script_content.rfind('"', 0, key_index) + 1
# 
#                 # Agora achamos o final da url
#                 end_index = script_content.find('"', key_index)
# 
#                 # E agora pegamos a URL (substring)
#                 url = script_content[start_index: end_index]
# 
#                 # Com a URL, entramos na página e descobrimos o título dela
#                 response, content = getpage(url)
#                 soup2 = parsepage(content)
#                 title = soup2.find('h1', class_='content-head__title').string
# 
#                 news.append(dict(title=title, link=url))
# 
#                 # Preparamos o próximo índice de busca
#                 i = key_index + 10
# 
#     return news


# Path: app/news_source_technology.py
from app.aml_utils import getpage, parsepage, get_ns
from app.news_source_national import g1

    return news


def __tec_conv(soup):
    """
    Gets the most read news from the Convergencia Digital page
    :param soup: the BeautifulSoup object
    :return: a list with the most read news from the Convergencia Digital page
    """
    news = []
    ns = get_ns('tec_conv')

    anchors = soup.find('div', id='slideshow').find_all('a')

    for a in anchors:
        if a.h1:
            title = a.h1.string
            link = 'http://www.convergenciadigital.com.br' + a['href']
            news.append(dict(title=title, link=link))
    return news


def __tec_canal(soup):
    """
    DEPRECATED
    Gets the most read news from the Canaltech page
    :param soup: the BeautifulSoup object
    :return: a list with the most read news from the Canaltech page
    """
    news = []

    ns = get_ns('tec_canal')