<commit_before>#coding=utf-8
import urllib.request
from lxml import etree
import myproxy
import requests
import re
import os
import json

class WechatSogouException(Exception):
    """基于搜狗的微信公众号爬虫异常类
    """
    pass

class WechatSogouVcodeException(WechatSogouException):
    """基于搜狗的微信公众号爬虫 出现验证码 异常类
    """
    pass

class Session(object):
    """基于搜狗的微信公众号爬虫COOKIE管理类

    Attributes:
        cookiefile: cookie存放文件名
        session： requests库中requests.session()对象，
    """

    def __init__(self):
        self.cookiefile = 'cookie.dat'

    def __enter__(self):
        """读取更新或生成session

        Returns:
            session

        Raises:
            WechatSogouException: session读取或生成错误
        """
        if os.path.exists(self.cookiefile):
            self.session = requests.session()
            with open(self.cookiefile) as f:
                cookie = json.load(f)
            self.session.cookies.update(cookie)
            try:
                WechatSpider().search_gzh('newsbro')
                return self.session
            except WechatSogouVcodeException:
                os.remove(self.cookiefile)
        self.session = requests.session()
        try:
            WechatSpider().search_gzh('newsbro')
        except WechatSogouVcodeException:
            raise WechatSogouVcodeException('session get error')
        return self.session

    def __exit__(self, exception_type, exception_value, exception_traceback):
        """存储session

        Raises:
            WechatSogouException: 没有session属性
        """
        if hasattr(self, 'session'):
            with open(self.cookiefile, 'w') as f:
                json.dump(self.session.cookies.get_dict(), f)
        else:
            raise WechatSogouException('session empty cannot save')

class WechatSpider(object):
    """基于搜狗的微信公众号爬虫类

    Attributes:
        agent: 模拟浏览器头
    """

    def __init__(self):
        self.agent = [
            'Opera/9.27 (Windows NT 5.2; U; zh-cn)',
            'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13',
            'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13',
            'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36',
            'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5'
        ]

    def __get_elem_text(self, elem):
        """抽取lxml.etree库中elem对象中文字

        Args:
            elem: lxml.etree库中elem对象

        Returns:
            elem中文字
        """
        rc = []
        for node in elem.itertext():
            rc.append(node.strip())
        return ''.join(rc)

    def __get_encoding_from_reponse(self, r):
        """获取requests库get或post返回的对象编码

        Args:
            r: requests库get或post返回的对象

        Returns:
            对象编码
        """
        encoding = requests.utils.get_encodings_from_content(r.text)
        return encoding[0] if encoding else requests.utils.get_encoding_from_headers(r.headers)

    def get_session(self):
        """获取session

        Returns:
            session: requests的session对象，有get或post方法
        """
        with Session() as self.session:
            pass

    def __get(self, url, host='', referer='', proxy=False):
        """封装request库get方法

        Args:
            url: 请求url
            host: 请求host
            referer: 请求referer
            proxy: 是否启用代理请求

        Returns:
            text: 请求url的网页内容

        Raises:
            WechatSogouException: 操作频繁以致出现验证码或requests请求返回码错误
        """
        headers = {
            "User-Agent": 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/45.0.2454.101 Chrome/45.0.2454.101 Safari/537.36',
            "Referer": referer if referer else 'http://weixin.sogou.com/',
            'Host': host if host else 'weixin.sogou.com',
        }
        req = self.session if hasattr(self, 'session') else requests
        if proxy:
            proip_http  = myproxy.get_one('http')
            proip_https = myproxy.get_one('https')
            proxies = {
                'http' : proip_http['http']+"://" + proip_http['ip'] + ":" + proip_http['duan'],
                'https' : proip_http['http'] + "://" + proip_http['ip'] + ":" + proip_http['duan']
            }
            r = req.get(url, headers=headers, proxies=proxies)
        else:
            r = req.get(url, headers=headers)
        if r.status_code == requests.codes.ok:
            r.encoding = self.__get_encoding_from_reponse(r)
            if '用户您好，您的访问过于频繁，为确认本次访问为正常用户行为，需要您协助验证' in r.text:
                #raise WechatSogouException('weixin.sogou.com verification code')
                raise WechatSogouVcodeException('weixin.sogou.com verification code')
        else:
            raise WechatSogouException('requests status_code error')
        return r.text

    def __replace_html(self, s):
        """替换html‘&quot;’等转义内容为正常内容

        Args:
            s: 文字内容

        Returns:
            s: 处理反转义后的文字
        """
        s = s.replace('&quot;','"')
        s = s.replace('&amp;','&')
        s = s.replace('amp;','')
        s = s.replace('&lt;','<')
        s = s.replace('&gt;','>')
        s = s.replace('&nbsp;',' ')
        s = s.replace(r"\\", r'')
        return s

    def search_gzh(self, name, page=1):
        """通过搜狗搜索获取关键字返回的文本

        Args:
            name: 搜索关键字
            page: 搜索的页数

        Returns:
            text: 返回的文本
        """
        request_url = 'http://weixin.sogou.com/weixin?query='+name+'&_sug_type_=&_sug_=n&type=1&page='+str(page)+'&ie=utf8'
        text = self.__get(request_url)
        return text

    def search_gzh_info(self, name, page):
        """对搜索的文本进行处理

        Args:
            name: 搜索关键字
            page: 搜索的页数

        Returns:
            列表，每一项均是{'name':name,'wechatid':wechatid,'jieshao':jieshao,'renzhen':renzhen,'qrcode':qrcodes,'img':img,'url':url}
            name: 公众号名称
            wechatid: 公众号id
            jieshao: 介绍
            renzhen: 认证，为空表示未认证
            qrcode: 二维码
            img: 头像图片
            url: 最近文章地址
        """<commit_msg>删除Session类，没卵用<commit_after>#coding=utf-8
import urllib.request
from lxml import etree
import myproxy
import requests
import re
import os
import json
import random

class WechatSogouException(Exception):
    """基于搜狗的微信公众号爬虫异常类
    """
    pass

class WechatSogouVcodeException(WechatSogouException):
    """基于搜狗的微信公众号爬虫 出现验证码 异常类
    """
    pass

class WechatSpider(object):
    """基于搜狗的微信公众号爬虫类

    Attributes:
        agent: 模拟浏览器头
    """

    def __init__(self):
        self.cookiefile = 'cookie.dat'
        self.agent = [
            'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
        ]

    def __get_elem_text(self, elem):
        """抽取lxml.etree库中elem对象中文字

        Args:
            elem: lxml.etree库中elem对象

        Returns:
            elem中文字
        """
        rc = []
        for node in elem.itertext():
            rc.append(node.strip())
        return ''.join(rc)

    def __get_encoding_from_reponse(self, r):
        """获取requests库get或post返回的对象编码

        Args:
            r: requests库get或post返回的对象

        Returns:
            对象编码
        """
        encoding = requests.utils.get_encodings_from_content(r.text)
        return encoding[0] if encoding else requests.utils.get_encoding_from_headers(r.headers)

    def __get(self, url, host='', referer='', proxy=False):
        """封装request库get方法

        Args:
            url: 请求url
            host: 请求host
            referer: 请求referer
            proxy: 是否启用代理请求

        Returns:
            text: 请求url的网页内容

        Raises:
            WechatSogouException: 操作频繁以致出现验证码或requests请求返回码错误
        """
        headers = {
            "User-Agent": self.agent[random.randint(0, len(self.agent) - 1)],
            "Referer": referer if referer else 'http://weixin.sogou.com/',
            'Host': host if host else 'weixin.sogou.com',
        }
s        if proxy:
            proip_http  = myproxy.get_one('http')
            proip_https = myproxy.get_one('https')
            proxies = {
                'http' : proip_http['http']+"://" + proip_http['ip'] + ":" + proip_http['duan'],
                'https' : proip_http['http'] + "://" + proip_http['ip'] + ":" + proip_http['duan']
            }
            r = requests.get(url, headers=headers, proxies=proxies)
        else:
            r = req.get(url, headers=headers)
        if r.status_code == requests.codes.ok:
            r.encoding = self.__get_encoding_from_reponse(r)
            if '用户您好，您的访问过于频繁，为确认本次访问为正常用户行为，需要您协助验证' in r.text:
                #raise WechatSogouException('weixin.sogou.com verification code')
                raise WechatSogouVcodeException('weixin.sogou.com verification code')
        else:
            raise WechatSogouException('requests status_code error')
        return r.text

    def __replace_html(self, s):
        """替换html‘&quot;’等转义内容为正常内容

        Args:
            s: 文字内容

        Returns:
            s: 处理反转义后的文字
        """
        s = s.replace('&quot;','"')
        s = s.replace('&amp;','&')
        s = s.replace('amp;','')
        s = s.replace('&lt;','<')
        s = s.replace('&gt;','>')
        s = s.replace('&nbsp;',' ')
        s = s.replace(r"\\", r'')
        return s

    def search_gzh(self, name, page=1):
        """通过搜狗搜索获取关键字返回的文本

        Args:
            name: 搜索关键字
            page: 搜索的页数

        Returns:
            text: 返回的文本
        """
        request_url = 'http://weixin.sogou.com/weixin?query='+urllib.request.quote(name)+'&_sug_type_=&_sug_=n&type=1&page='+str(page)+'&ie=utf8'
        text = self.__get(request_url)
        return text

    def search_gzh_info(self, name, page):
        """对搜索的文本进行处理

        Args:
            name: 搜索关键字
            page: 搜索的页数

        Returns:
            列表，每一项均是{'name':name,'wechatid':wechatid,'jieshao':jieshao,'renzhen':renzhen,'qrcode':qrcodes,'img':img,'url':url}
            name: 公众号名称
            wechatid: 公众号id
            jieshao: 介绍
            renzhen: 认证，为空表示未认证
            qrcode: 二维码
            img: 头像图片
            url: 最近文章地址
        """
{'fraction_non_alphanumeric': 0.1246450880181715, 'fraction_numerical': 0.02015900056785917, 'mean_word_length': 1.9360933592664629, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5166745', 'n_tokens_mistral': 4063, 'n_tokens_neox': 4091, 'n_words': 652}