import urllib2, cookielib
from bs4 import BeautifulSoup
import json
import time
from random import uniform
from get_search_results import get_soup




def add_article_texts(search_results):
    """For each search result, get the page's text and add to search_results.
    search_results is in the format generated by get_search_results.py
    """
    n = len(search_results)
    # start = time.clock()
    for i, pain in enumerate(search_results):
        print "\n\nReading articles for %s (%d/%d):" % (pain, i+1, n)
        # print "Elapsed time so far: %d seconds" % (time.clock()-start) 
        for i, item in enumerate(search_results[pain][1]):
            
            url = item[1]
            print "Current url:", url
            
            soup = get_soup(url)
            text = get_page_text(soup)

            search_results[pain][1][i].append(text)


def get_page_text(soup):
    """Return all paragraph text of a webpage in a single string.
    """
    if soup is None:
        return ''
    paragraphs = [para.text for para in soup.select('p')]
    text = '\n'.join(paragraphs)
    return text


if __name__ == '__main__':

    with open('../data/outputs/search_results.txt', 'r') as json_data:
        search_results = json.load(json_data)

        # t0 = time.clock()
        # add_article_texts(search_results)
        # t1 = time.clock()
        # print "Finished grabbing articles. Elapsed minutes =", (t1-t0)/60

    with open('../data/outputs/search_results_and_texts.txt', 'w') as outfile:
        json.dump(search_results, outfile)

    # print get_page_text('http://entnemdept.ufl.edu/creatures/urban/medical/saddleback_caterpillar.htm')
