<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Created on 2017-02-21
# Project: basespider

import json
import time
import socket
import requests
import traceback
from requests.exceptions import ReadTimeout
from requests.exceptions import ConnectionError


class BaseGenerator(object):
    """
    BaseSpider Generator
    """

    def __init__(self):
        """
        Generator Initialization
        """
        pass

    def generate(self, url):
        """
        Obtain URI
        :return:
        """
        return {"url": url, "args": 'None'}

    def start_generator(self):
        """
        Start Generator
        :return:
        """
        start_url = "__START_URL__"
        result = self.generate(start_url)
        return result


class BaseDownloader(object):
    """
    BaseSpider Downloader
    """

    def __init__(self):
        """
        Downloader Initialization
        """
        pass

    def download(self, request):
        """
        Downloader Download By tools
        :return: response object
        """

        tools = request.get('tools', 'requests')
        if tools == "requests":
            self.reqst = requests.Session()
            self.headers =
            {'Accept': 'text/html, application/xhtml+xml, */*',
             'Accept-Encoding': 'gzip, deflate',
             'Accept-Language': 'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
             'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:39.0) Gecko/20100101 Firefox/39.0'}

            url = request.get('url')
            method = request.get('method', 'GET')
            kwargs = {
                'headers': request.get('headers', self.headers),
                'cookies': request.get('cookies', None),
                'proxies': request.get('proxies', None),
                'timeout': request.get('timeout', 30)}
            if str(method).upper() == 'GET':
                kwargs['params'] = request.get('params', {})
            elif str(method).upper() == 'POST':
                kwargs['data'] = request.get('data', {})
            try:
                resp = self.reqst.request(method=method, url=url, **kwargs)
                return resp
            except Exception:
                print traceback.format_exc()
                raise Exception

        elif tools == 'phantomjs':
            """
            Download by Phantomjs
            """
            # TODO
            pass



class BaseParser(object):
    """
    BaseSpider Parser
    """

    def __init__(self):
        """
        Parser Initialization
        """
        pass

    def parser(self, resp):
        """
        Paeser resp content
        :param resp:
        :return:
        """
        return resp


<commit_msg>update basespider args<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Created on 2017-02-21
# Project: basespider

import json
import time
import socket
import requests
import traceback
from requests.exceptions import ReadTimeout
from requests.exceptions import ConnectionError


class BaseGenerator(object):
    """
    BaseSpider Generator
    """

    def __init__(self):
        """
        Generator Initialization
        """
        pass

    def generate(self, url):
        """
        Obtain URI
        :return:
        """
        return {"url": url, "args": 'None'}

    def start_generator(self):
        """
        Start Generator
        :return:
        """
        start_url = "__START_URL__"
        result = self.generate(start_url)
        return result


class BaseDownloader(object):
    """
    BaseSpider Downloader
    """

    def __init__(self):
        """
        Downloader Initialization
        """
        pass

    def download(self, url, args={}, tools='requests', method='GET'):
        """
        Downloader Download By tools
        :return: response object
        """


        if tools == "requests":
            self.reqst = requests.Session()
            self.headers =
            {'Accept': 'text/html, application/xhtml+xml, */*',
             'Accept-Encoding': 'gzip, deflate',
             'Accept-Language': 'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
             'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:39.0) Gecko/20100101 Firefox/39.0'}
            kwargs = {
                'headers': args.get('headers', self.headers),
                'cookies': args.get('cookies', None),
                'proxies': args.get('proxies', None),
                'timeout': args.get('timeout', 30)}
            if str(method).upper() == 'GET':
                kwargs['params'] = args.get('params', {})
            elif str(method).upper() == 'POST':
                kwargs['data'] = args.get('data', {})
            try:
                resp = self.reqst.request(method=method, url=url, **kwargs)
                return resp
            except Exception:
                print traceback.format_exc()
                raise Exception

        elif tools == 'js':
            """
            Download by Phantomjs
            """
            # TODO
            pass



class BaseParser(object):
    """
    BaseSpider Parser
    """

    def __init__(self):
        """
        Parser Initialization
        """
        pass

    def parser(self, resp):
        """
        Paeser resp content
        :param resp:
        :return:
        """
        return resp



{'fraction_non_alphanumeric': 0.1326171513243193, 'fraction_numerical': 0.014447119837006852, 'mean_word_length': 1.6706231454005935, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3089744', 'n_tokens_mistral': 1576, 'n_tokens_neox': 1483, 'n_words': 345}