<commit_before>#!/usr/bin/python2.6  
# -*- coding: utf-8 -*-  
#encoding=utf-8

'''
Created on 2014年5月27日

@author: kerry
'''
import httplib
from StringIO import  StringIO
from gzip import GzipFile

class MIGHttpResponse:
    '''
    classdocs
    '''
    __url = ''
    __host =''
    __data = ''

    def __init__(self, url,host):
        '''
        Constructor
        '''
        self.__url = url
        self.__host = host
        
    def HttpMethodGet(self,header={}):
        conn = httplib.HTTPConnection(self.__host)
        headers = {
            "Host": self.__host,
            "Connection": "keep-alive",
            "Cache-Control": "max-age=0",
            "accept_charset":"utf-8,gbk, *;q=0.7",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
            "User-Agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36",
            "ContentType":"application/x-www-form-urlencoded" 
        }
        conn.request(method="GET", url=self.__url,headers = headers)
        response = conn.getresponse()
        if(response.getheader('content-encoding') == 'gzip'):
            data = response.read()
            data2 = GzipFile('','r',0,StringIO(data)).read()
            data = data2
        else:
            data = data = response.read()
        
        self.__data = data
    
    def HttpGetContent(self):
        print self.__data
    
        
        
    
        <commit_msg>+ http Post 请求处理 + http 加入自定义http头和cookie机制<commit_after>#!/usr/bin/python2.6  
# -*- coding: utf-8 -*-  
#encoding=utf-8

'''
Created on 2014年5月27日

@author: kerry
'''
import httplib
import urllib
from StringIO import  StringIO
from gzip import GzipFile

#http base
#若没有自定http头则默认以为chrome浏览器方式
class MIGHttpBase:
    def __init__(self,url,host):
        self.url = url
        self.host = host
        self.data = ''
        self.headers = {
            "Host": self.host,
            "Connection": "keep-alive",
            "Cache-Control": "max-age=0",
            "accept_charset":"utf-8,gbk, *;q=0.7",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
            "User-Agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36",
            "ContentType":"application/x-www-form-urlencoded" 
        }
       
    def HttpGetContent(self):
        return self.data


class MIGHttpMethodPost(MIGHttpBase):
    
    def __init__(self,url,host):
        MIGHttpBase.__init__(self, url, host)
        
    def HttpMethodPost(self,data,header={},cookies={}):
        conn = httplib.HTTPConnection(self.host)
        print data
        data_urlencode = urllib.urlencode(data)
        conn.request(method="POST", url=self.url, body = data_urlencode, headers = self.headers)
        response = conn.getresponse()
        if(response.getheader('content-encoding') == 'gzip'):
            data = response.read()
            data2 = GzipFile('','r',0,StringIO(data)).read()
            data = data2
        else:
            data = data = response.read()
        self.data = data
        
        
    
class MIGHttpMethodGet(MIGHttpBase):
    '''
    classdocs
    '''
    def __init__(self,url,host):
        MIGHttpBase.__init__(self, url, host)
        
    def HttpMethodGet(self,header={},cookies={}):
        if(len(header)>0):
            headers = header
        else:
            headers = self.headers
        
        if(len(cookies)>0):
            headers["Cookies"] = cookies
            
        conn = httplib.HTTPConnection(self.host)
        print headers
        conn.request(method="GET", url=self.url,headers = headers)
        response = conn.getresponse()
        if(response.getheader('content-encoding') == 'gzip'):
            data = response.read()
            data2 = GzipFile('','r',0,StringIO(data)).read()
            data = data2
        else:
            data = data = response.read()
        self.data = data

    
        
        
    
        
{'fraction_non_alphanumeric': 0.14435196195005945, 'fraction_numerical': 0.02497027348394768, 'mean_word_length': 2.0302593659942363, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6145858', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1378, 'n_words': 229}