<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'Alien'
__version__ = '1.0'
__doc__ = '''
Simple Wikipedia Crawler

A simple crawler for Wikipedia.
Given a page title it generates
a list of links found in that page and
crawls the first one.
'''

import wikipedia
import codecs
import Queue


class Crawler:
    def __init__(self):

        self.depth_counter = 0
        self.tested_pages = set()
        self.list_links = set()
        self.links_queue = Queue.Queue(0)


    def crawl(self, link, depth):

        if depth < self.depth_counter:
            return

        file_link = codecs.open(link.title() + ' links.txt', 'w', 'utf-8')

        for url in self.get_links(link):
            self.links_queue.put(url)
            print (link.title() + ' --> ' + url)
            file_link.write(link.title() + ' --> ' + url + '\n')

        if depth != 0:
            self.depth_counter += 1

        try:
            self.crawl(self.links_queue._get(), depth)
        except (IOError, RuntimeError):
            print 'Some kind of error happened'

        except (wikipedia.DisambiguationError):
            print 'Disambiguation Error'
            print wikipedia.DisambiguationError.__unicode__()

        except (wikipedia.PageError):
            print wikipedia.PageError.__unicode__()
            self.crawl(self.links_queue._get(), depth)

    def get_links(self, wiki_page):

        if wiki_page.title() in self.tested_pages:
            raise IOError('Link already found')

        self.tested_pages.add(wiki_page.title())

        wiki_links = wikipedia.WikipediaPage(wiki_page.title()).links

        return wiki_links


def main():

    WikiCrawler = Crawler()

    input_wiki_page = raw_input('Which Wikipedia page should I start from? ')

    try:
        crawling_depth = int(raw_input('Tell me a number to let me know how deep the rabbit hole should I go. '))
    except ValueError:
        crawling_depth = 0

    try:

        WikiCrawler.crawl(input_wiki_page, crawling_depth)

    except KeyboardInterrupt:
        print 'Program stopped.'
    except StandardError:
        print 'End!'



if __name__ == '__main__':
    main()<commit_msg>Added a prompt that ask the user if he wants to donate to the Wikimedia Foundation and promptly opens the donation page in the default browser.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'Alien'
__version__ = '1.1'
__doc__ = '''
Simple Wikipedia Crawler

A simple crawler for Wikipedia.
Given a page title it generates
a list of links found in that page and
crawls the first one.
'''

import wikipedia
import codecs
import Queue


class Crawler:
    def __init__(self):

        self.depth_counter = 0
        self.tested_pages = set()
        self.list_links = set()
        self.links_queue = Queue.Queue(0)


    def crawl(self, link, depth):

        if depth < self.depth_counter:
            return

        file_link = codecs.open(link.title() + ' links.txt', 'w', 'utf-8')

        for url in self.get_links(link):
            self.links_queue.put(url)
            print (link.title() + ' --> ' + url)
            file_link.write(link.title() + ' --> ' + url + '\n')

        if depth != 0:
            self.depth_counter += 1

        try:
            self.crawl(self.links_queue._get(), depth)
        except (IOError, RuntimeError):
            print 'Some kind of error happened'

        except (wikipedia.DisambiguationError):
            print 'Disambiguation Error'
            print wikipedia.DisambiguationError.__unicode__()

        except (wikipedia.PageError):
            print wikipedia.PageError.__unicode__()
            self.crawl(self.links_queue._get(), depth)

    def get_links(self, wiki_page):

        if wiki_page.title() in self.tested_pages:
            raise IOError('Link already found')

        self.tested_pages.add(wiki_page.title())

        wiki_links = wikipedia.WikipediaPage(wiki_page.title()).links

        return wiki_links


def main():

    WikiCrawler = Crawler()

    input_wiki_page = raw_input('Which Wikipedia page should I start from? ')

    try:
        crawling_depth = int(raw_input('Tell me a number to let me know how deep the rabbit hole should I go. '))
    except ValueError:
        crawling_depth = 0

    try:

        WikiCrawler.crawl(input_wiki_page, crawling_depth)

    except KeyboardInterrupt:
        print 'Program stopped.'

    print '\n'

    donation_answer = raw_input("This project wouldn't have been possible without the existence of the Wikimedia Foundation."
                                "Do you want to make a donation to them? (Write Y or N)")

    donation_answer = donation_answer.upper()

    if donation_answer == 'Y' or donation_answer == 'YES':
        wikipedia.donate()

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.10223509933774834, 'fraction_numerical': 0.0037251655629139072, 'mean_word_length': 2.4011259676284307, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6829044', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1392, 'n_words': 406}