# Path: QiuBaiSpider/tools.py
# class Tools(object):
#     def get_current_time(format_="[%Y-%m-%d %H:%M:%S]"):
#         return time.strftime(format_, time.localtime(time.time()))
# 
#     def log(msg=None):
#         print(Tools.get_current_time() + msg)
# 
#     def wash_off_html_tag(str=None):
#         if str is None:
#             return str
#         return re.sub(re.compile(r'<\w+>|</\w+>|<\w+/>|&quot'), '', str)
# 
#     def setup_log_mode(use_file=False, file_name="record.log"):
#         if use_file:
#             sys.stdout = open(file_name, "w")


import re
from urllib import request
from QiuBaiSpider.tools import Tools


class PageItems(object):
    def __init__(self, page=None):
        self.page = page

    def get_page_dict_items(self):
        if self.page is None:
            return None
        try:
            url = "http://www.qiushibaike.com/hot/page/" + str(self.page)
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.100 Safari/537.36"
            }
            req = request.Request(url, headers=headers)
            content = request.urlopen(req).read().decode("utf-8")

            pattern = re.compile(r'<div class="author clearfix">.*?<h2>(.*?)</h2>.*?<div class="content">(.*?)</div>', re.S)
            items = re.findall(pattern, content)
            dict_list = list()
            for item in items:

# Path: QiuBaiSpider/tools.py
# class Tools(object):
#     def get_current_time(format_="[%Y-%m-%d %H:%M:%S]"):
#         return time.strftime(format_, time.localtime(time.time()))
# 
#     def log(msg=None):
#         print(Tools.get_current_time() + msg)
# 
#     def wash_off_html_tag(str=None):
#         if str is None:
#             return str
#         return re.sub(re.compile(r'<\w+>|</\w+>|<\w+/>|&quot'), '', str)
# 
#     def setup_log_mode(use_file=False, file_name="record.log"):
#         if use_file:
#             sys.stdout = open(file_name, "w")


# Path: QiuBaiSpider/page_items.py
import re
from urllib import request
from QiuBaiSpider.tools import Tools



class PageItems(object):
    def __init__(self, page=None):
        self.page = page

    def get_page_dict_items(self):
        if self.page is None:
            return None
        try:
            url = "http://www.qiushibaike.com/hot/page/" + str(self.page)
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.100 Safari/537.36"
            }
            req = request.Request(url, headers=headers)
            content = request.urlopen(req).read().decode("utf-8")

            pattern = re.compile(r'<div class="author clearfix">.*?<h2>(.*?)</h2>.*?<div class="content">(.*?)</div>', re.S)
            items = re.findall(pattern, content)
            dict_list = list()
            for item in items:

                temp = {"name": Tools.wash_off_html_tag(item[0]), "content": Tools.wash_off_html_tag(item[1])}