from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
import networkx as nx
import matplotlib.pyplot as plt

DEPTH_MAX = 2
adjlists = {}
G = nx.Graph()
E = nx.Graph()
totalwebpagecount = 0
csvfile = open("static/links.txt", "a")
csvfile_clear = open("static/links.txt", "r+")

def webcrawl(url, adjlists, depth):
    global DEPTH_MAX
    global totalwebpagecount
    global G
    global E
    global csvfile
    G.add_node(url)

    try:
        req = Request(url)
        html_page = urlopen(req)
    except AttributeError:
        return 0
    except ValueError:
        return 0
    except:
        return 0

    # at this point, successful open
    print('total before check:', totalwebpagecount)
    if totalwebpagecount >= 100:
        return 1
    if depth > DEPTH_MAX:
        return 1

    soup = BeautifulSoup(html_page, "lxml")

    links = []
    for link in soup.findAll('a'):
        testlink = link.get('href')
        try:
            if testlink[:8] == 'https://':
                links.append(testlink)
        except TypeError:
            pass

    if len(links) == 0:
        return 1

    adjlists.update({url: links})

    webpagecount = 1
    for link in links:
        G.add_edge(url, link)
        webpagecount += webcrawl(link, adjlists, depth + 1)
        totalwebpagecount += webpagecount
        print('totalwebpagecount:', totalwebpagecount)
        print(nx.info(G))

        csvfile.write(str(link) + " : " + str(links))
        csvfile.write("\n")

        nx.draw(E)
        nx.draw(G, with_labels=True, node_size=0, font_size=5)
        plt.savefig('static/graph.png', transparent=True, format='PNG')
        plt.show()

    return webpagecount


def main(url):
    global adjlists
    global csvfile_clear

    csvfile_clear.truncate(0)
    print('this is url:', url)
    webpagecount = webcrawl(url, adjlists, 0)

    print('this is webpagecount:', webpagecount)