<reponame>tfederico/WatsonInnovation<filename>Backend/download.py
# coding: utf-8

#Searching and Downloading Google Images/Image Links

#Import Libraries

#coding: UTF-8

import time       #Importing the time library to check the time of code execution
import sys    #Importing the System Library
import os
import urllib2
import json
import argparse
from urllib2 import Request,urlopen
from urllib2 import URLError, HTTPError

pause = 0

#Downloading entire Web Document (Raw Page Content)
def download_page(url):
    version = (3,0)
    cur_version = sys.version_info
    if cur_version >= version:     #If the Current Version of Python is 3.0 or above
        import urllib.request    #urllib library for Extracting web pages
        try:
            headers = {}
            headers['User-Agent'] = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
            req = urllib.request.Request(url, headers = headers)
            resp = urllib.request.urlopen(req)
            respData = str(resp.read())
            return respData
        except Exception as e:
            print(str(e))
    else:                        #If the Current Version of Python is 2.x
        import urllib2
        try:
            headers = {}
            headers['User-Agent'] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
            req = urllib2.Request(url, headers = headers)
            response = urllib2.urlopen(req)
            page = response.read()
            return page
        except:
            return"Page Not found"


#Finding 'Next Image' from the given raw page
def _images_get_next_item(s):
    start_line = s.find('rg_di')
    if start_line == -1:    #If no links are found then give an error!
        end_quote = 0
        link = "no_links"
        return link, end_quote
    else:
        start_line = s.find('"class="rg_meta"')
        start_content = s.find('"ou"',start_line+1)
        end_content = s.find(',"ow"',start_content+1)
        content_raw = str(s[start_content+6:end_content-1])
        return content_raw, end_content


#Getting all links with the help of '_images_get_next_image'
def _images_get_all_items(page):
    items = []
    while True:
        item, end_content = _images_get_next_item(page)
        if item == "no_links":
            break
        else:
            items.append(item)      #Append all the links in the list named 'Links'
            time.sleep(pause)        #Timer could be used to slow down the request for image downloads
            page = page[end_content:]
    return items

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Downloading images based on painting and author names.')
    parser.add_argument('-number', '--n', help="Number of painting to download", required=True, type=int)
    parser.add_argument('-paint', '--p', help="Painting name", required=True,type=str)
    parser.add_argument('-author', '--a', help="Author info", required=True,type=str)
    

    args = parser.parse_args()

    paint = args.p.replace("_"," ") #Work-around for Java issues
    auth = args.a.split('_')
    n_max = args.n

    #This list is used to search keywords. You can edit this list to search for google images of your choice. 
    #You can simply add and remove elements of the list.
    search_keyword = [paint]

    #This list is used to further add suffix to your search term. Each element of the list will help you download 100 images. 
    #First element is blank which denotes that no suffix is added to the search keyword of the above list. You can edit the list by adding/deleting
    #elements from it.So if the first element of the search_keyword is 'Australia' and the second element of keywords is 'high resolution', 
    #then it will search for 'Australia High Resolution'
    keywords = [' painting']+auth

    n_max = n_max * len(keywords)
      

    ############## Main Program ############
    t0 = time.time()   #start the timer

    #Download Image Links
    i= 0
    while i<len(search_keyword):
        items = []
        iteration = "Item no.: " + str(i+1) + " -->" + " Item name = " + str(search_keyword[i])
        print (iteration)
        print ("Evaluating...")
        search_keywords = search_keyword[i]
        search = search_keywords.replace(' ','%20')
        
        #make a search keyword  directory
        try:
            os.makedirs("res/images/"+search_keywords.replace(' ','_'))
        except OSError as e:
            if e.errno != 17:
                raise   
            # time.sleep might help here
            pass
        
        j = 0
        while j<len(keywords):
            pure_keyword = keywords[j].replace(' ','%20')
            url = 'https://www.google.com/search?q=' + search + pure_keyword + '&espv=2&biw=1366&bih=667&site=webhp&source=lnms&tbm=isch&sa=X&ei=XosDVaCXD8TasATItgE&ved=0CAcQ_AUoAg'
            raw_html =  (download_page(url))
            time.sleep(pause)
            items = items + (_images_get_all_items(raw_html))
            j = j + 1
        #print ("Image Links = "+str(items))
        print ("Total Image Links = "+str(len(items)))
        print ("\n")


        #This allows you to write all the links into a test file. This text file will be created in the same directory as your code. 
        #You can comment out the below 3 lines to stop writing the output to the text file.
        #info = open('output.txt', 'a')        #Open the text file called database.txt
        #info.write(str(i) + ': ' + str(search_keyword[i-1]) + ": " + str(items) + "\n\n\n")         #Write the title of the page
        #info.close()                            #Close the file

        t1 = time.time()    #stop the timer
        total_time = t1-t0   #Calculating the total time required to crawl, find and download all the links of 60,000 images
        print("Total time taken: "+str(total_time)+" Seconds")
        print ("Starting Download...")

        ## To save imges to the same directory
        # IN this saving process we are just skipping the URL if there is any error

        k=0
        errorCount=0
        
        l=0
        #print limit
        while(k<len(items) and l<n_max):

            try:
                req = Request(items[k], headers={"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"})
                response = urlopen(req,None,15)
                
                #print(response.info())
                
                ext = ""
                for item in str(response.info()).split('\n'):
                    if "Content-Type" in item and "image" in item and "webp" not in item and "gif" not in item and "bmp" not in item:
                        #print item
                        ext = item.split("/")[1]

                if ext == "":
                    k=k+1
                    response.close()
                    continue    

                if "jpeg" in ext or "JPEG" in ext: 
                    ext = "jpg"
                
                ext = ext.strip()
                output_file = open("res/images/"+search_keywords.replace(' ','_')+"/"+str(l+1)+"."+ext,'wb')
                

                data = response.read()
                
                output_file.write(data)
                response.close()

                #print("completed ====> "+str(k+1))

                k=k+1
                l=l+1

            except IOError:   #If there is any IOError

                errorCount+=1
                print("IOError on image "+str(l+1))
                k=k+1
                if(l > 0):
                    l=l-1

            except HTTPError as e:  #If there is any HTTPError

                errorCount+=1
                print("HTTPError"+str(k))
                k=k+1
                if(l > 0):
                    l=l-1

            except URLError as e:

                errorCount+=1
                print("URLError "+str(k))
                k=k+1
                if(l > 0):
                    l=l-1

        i = i+1

    print("\n")
    print("Everything downloaded!")
    print("\n"+str(errorCount)+" ----> total Errors")

    #----End of the main program ----#
