<commit_before># -*- coding: utf-8 -*-

import os
import urllib, urllib2, urlparse
import Cookie
from cookielib import CookieJar
import base64

"""
Simple module to request HTTP

JinnLynn
http://jeeker.net

get(url, **kwargs)
post(url, **kwargs)
download(url, **kwargs)

Request(
    'http://jeeker.net',
    data = {},
    type = 'GET',               # GET POST default:GET
    referer = '',
    user_agent = '',
    cookie = None,              # CookieJar, Cookie.S*Cookie, dict, string
    auth = {'usr':'', 'pwd':''}, # Only Basic Authorization
    debug = False
    )
"""

_DEFAULT_TIMEOUT = 90

def get(url, **kwargs):
    kwargs.update(type='GET')
    return Request(url, **kwargs)

def post(url, **kwargs):
    kwargs.update(type='POST')
    return Request(url, **kwargs)

def download(url, local, **kwargs):
    if not local:
        raise ValueError('local filepath is empty')
    try:
        res = Request(url, **kwargs)
        read_size = 0
        real_size = int(res.header['content-length'])
        with open(local, 'wb') as f:
            while True:
                block = res.response.read(1024*8)
                if not block:
                    break
                f.write(block)
                read_size += len(block)
            if read_size < real_size:
                raise urllib.ContentTooShortError(
                    'retrieval incomplete: got only {} out of {} bytes'.formate(read_size, real_size),
                    None
                    )
    except Exception, e:
        raise e

class Request(object):
    def __init__(self, url, **kwargs):
        self.request = None
        self.response = None
        self.code = -1
        self.info = {}
        self.cookieJar = None
        self.reason = ''

        data = kwargs.get('data', None)
        if data:
            if isinstance(data, dict):
                data = urllib.urlencode(data)
            if not isinstance(data, basestring):
                data = None
                raise ValueError('data must be string or dict')

        request_type = kwargs.get('type', 'POST')
        if data and isinstance(request_type, basestring) and request_type.upper()!='POST':
            url = '{}?{}'.format(url, data)
            data = None # GET data must be None

        self.request = urllib2.Request(url, data)<commit_msg>make dirs when download folder is non-existent<commit_after># -*- coding: utf-8 -*-

import os
import urllib, urllib2, urlparse
import Cookie
from cookielib import CookieJar
import base64

"""
Simple module to request HTTP

JinnLynn
http://jeeker.net

get(url, **kwargs)
post(url, **kwargs)
download(url, **kwargs)

Request(
    'http://jeeker.net',
    data = {},
    type = 'GET',               # GET POST default:GET
    referer = '',
    user_agent = '',
    cookie = None,              # CookieJar, Cookie.S*Cookie, dict, string
    auth = {'usr':'', 'pwd':''}, # Only Basic Authorization
    debug = False
    )
"""

_DEFAULT_TIMEOUT = 90

def get(url, **kwargs):
    kwargs.update(type='GET')
    return Request(url, **kwargs)

def post(url, **kwargs):
    kwargs.update(type='POST')
    return Request(url, **kwargs)

def download(url, local, **kwargs):
    if not local:
        raise ValueError('local filepath is empty')
    try:
        if not os.path.exists(os.path.dirname(local)):
            os.makedirs(os.path.dirname(local))
        res = Request(url, **kwargs)
        read_size = 0
        real_size = int(res.header['content-length'])
        with open(local, 'wb') as f:
            while True:
                block = res.response.read(1024*8)
                if not block:
                    break
                f.write(block)
                read_size += len(block)
        if read_size < real_size:
            raise urllib.ContentTooShortError(
                'retrieval incomplete: got only {} out of {} bytes'.formate(read_size, real_size),
                None
                )
    except Exception, e:
        raise e

class Request(object):
    def __init__(self, url, **kwargs):
        self.request = None
        self.response = None
        self.code = -1
        self.info = {}
        self.cookieJar = None
        self.reason = ''

        data = kwargs.get('data', None)
        if data:
            if isinstance(data, dict):
                data = urllib.urlencode(data)
            if not isinstance(data, basestring):
                data = None
                raise ValueError('data must be string or dict')

        request_type = kwargs.get('type', 'POST')
        if data and isinstance(request_type, basestring) and request_type.upper()!='POST':
            url = '{}?{}'.format(url, data)
            data = None # GET data must be None

        self.request = urllib2.Request(url, data)
{'fraction_non_alphanumeric': 0.1238474434199497, 'fraction_numerical': 0.00586756077116513, 'mean_word_length': 1.9887288666249217, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3552242', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1394, 'n_words': 383}