<commit_before>#!/usr/bin/env python
# encoding: utf-8

##
# @file zeus_crawler.py
# @brief Crawler
# @author 谢志杰/Shieh
# @version v1.0.0.0
# @date 2014-10-07


import os
import time
import requests
import optparse
from os.path import splitext, dirname, isdir, exists
from urlparse import urlparse, urljoin
from gevent.pool import Pool
from gevent import monkey, Timeout
from bs4 import BeautifulSoup
monkey.patch_all()


VERSION = 'v1.0.0'
USAGE = "usage: %prog [options] arg1 arg2"


def get_options():
    """TODO: Docstring for get_options.
    :returns: TODO

    """
    parser = optparse.OptionParser(usage=USAGE, version=VERSION)

    parser.add_option('-m', '--main', action='store', type='string',
                      help='Main Page.', dest='url')
    parser.add_option('-p', '--pool', action='store', type='int',
                      help='Size of pool.', dest='pool')
    parser.add_option('-t', '--test', action='store_true', dest='test',
                      help='Test the difference between sync and async.')
    parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
                      help='Output verbosely.')

    return parser.parse_args()


def log_time(fn):
    """TODO: Docstring for log_time.

    :fn: TODO
    :returns: TODO

    """
    def wrapper(*args, **kwargs):
        start_time = time.time()

        result = fn(*args, **kwargs)

        end_time = time.time()
        print_info('Time used : {0}'.format(end_time - start_time))

        return result

    return wrapper


def print_info(info):
    """TODO: Docstring for print_info.

    :info: TODO
    :returns: TODO

    """
    print '\033[;32m%s\033[0m' % (info)


def print_warning(warning):
    """TODO: Docstring for print_warning.

    :warning: TODO
    :returns: TODO

    """
    print '\033[;33m%s\033[0m' % (warning)<commit_msg>Remove the side-effect of decorator.<commit_after>#!/usr/bin/env python
# encoding: utf-8

##
# @file zeus_crawler.py
# @brief Crawler
# @author 谢志杰/Shieh
# @version v1.0.0.0
# @date 2014-10-07


import os
import time
import requests
import optparse
from os.path import splitext, dirname, isdir, exists
from functools import wraps
from urlparse import urlparse, urljoin
from bs4 import BeautifulSoup
from gevent.pool import Pool
from gevent import monkey, Timeout
monkey.patch_all()


VERSION = 'v1.0.0'
USAGE = "usage: %prog [options] arg1 arg2"


def get_options():
    """TODO: Docstring for get_options.
    :returns: TODO

    """
    parser = optparse.OptionParser(usage=USAGE, version=VERSION)

    parser.add_option('-m', '--main', action='store', type='string',
                      help='Main Page.', dest='url')
    parser.add_option('-p', '--pool', action='store', type='int',
                      help='Size of pool.', dest='pool')
    parser.add_option('-t', '--test', action='store_true', dest='test',
                      help='Test the difference between sync and async.')
    parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
                      help='Output verbosely.')

    return parser.parse_args()


def log_time(fn):
    """TODO: Docstring for log_time.

    :fn: TODO
    :returns: TODO

    """
    @wraps(fn)
    def wrapper(*args, **kwargs):
        start_time = time.time()

        result = fn(*args, **kwargs)

        end_time = time.time()
        print_info('Time used : {0}'.format(end_time - start_time))

        return result

    return wrapper


def print_info(info):
    """TODO: Docstring for print_info.

    :info: TODO
    :returns: TODO

    """
    print '\033[;32m%s\033[0m' % (info)


def print_warning(warning):
    """TODO: Docstring for print_warning.

    :warning: TODO
    :returns: TODO

    """
    print '\033[;33m%s\033[0m' % (warning)
{'fraction_non_alphanumeric': 0.1488630354309889, 'fraction_numerical': 0.02009518773135907, 'mean_word_length': 2.883983572895277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6225255', 'n_tokens_mistral': 1322, 'n_tokens_neox': 1236, 'n_words': 318}