# Path: app/tasks/crawler.py
# class BaseCrawler(metaclass=ABCMeta):
#     def __init__(self):
#         self.site = None
#         self.timeout = 30
#         self.session = requests.Session()
#         self.headers = {
#             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/'
#                           '60.0.3112.113 Safari/537.36'
#         }
#         self.today = datetime.datetime.today().strftime('%Y-%m-%d')
# 
#     def fetch_page(self, url, data=None, headers=None, timeout=None):
#         try:
#             timeout = timeout or self.timeout
#             headers = headers or self.headers
#             if data is None:
#                 r = self.session.get(url=url, timeout=timeout, headers=headers)
#             else:
#                 r = self.session.post(url=url, data=data, timeout=timeout, headers=headers)
#         except Exception as e:
#             print(str(e))
#         else:
#             return r
# 
#     def login(self):
#         cookies = self._login()
#         self.session.cookies = cookies
# 
#     def _login(self):
#         pass
# 
#     def parser(self):
#         pass
# 
#     def save(self, title, body, style='转载', category='Python 进阶', post_img=choice_img()):
#         app = create_app('default')
#         with app.app_context():
#             post = Post(title=title,
#                         body=body,
#                         style=style,
#                         category=category,
#                         author_id=1,
#                         post_img=post_img,
#                         created=datetime.datetime.now(),
#                         source=self.site)
#             db.session.add(post)
#             db.session.commit()
# 
#     def is_today(self, s):
#         return s == datetime.datetime.now().strftime('%Y-%m-%d')
# 
#     def start(self):
#         pass
# 
#     def upload_img(self, file_name, file, prefix, domian_name='https://static.51qinqing.com'):
#         u = UploadToQiniu(file, prefix, mark=True)
#         ret, info = u.upload_web(prefix + file_name, file)
#         key = ret['key']
#         url = domian_name + '/' + key
#         return url
# 
#     @abstractmethod
#     def parse(self):
#         pass
# 
# Path: app/shares.py
# def choice_img():
#     return 'https://static.51qinqing.com/postimg/' + random.choice(img_list)


from .crawler import BaseCrawler
from lxml import etree
from bs4 import BeautifulSoup
from ..shares import choice_img
import random
import datetime
                    i.decompose()
                foot = soup.find('div', style="font-family:'Helvetica Neue';font-size:14px;")
                next_ = soup.find('div', clas="zan-page bs-example")
                copy_right = soup.find('div', class_="copyright alert alert-success")
                foot.decompose()
                next_.decompose()
                copy_right.decompose()
                as_ = soup.find_all('a')
                for i in as_:
                    i.decompose()
                tree = etree.HTML(r.text)
                title = tree.xpath('//div[@class="title-article"]/h1/a/text()')[0]
                print(title)
                codes = soup.find_all('div',
                                      class_='crayon-syntax crayon-theme-solarized-light crayon-font-monaco crayon-os-pc print-yes notranslate')
                images = soup.find('div', class_='centent-article').find_all('img')
                img_list = []
                if images:
                    for i in images:
                        new_img = soup.new_tag('img')
                        new_img.attrs['width'] = '100%'
                        url = i.get('src')
                        if str(url).startswith('/wp-content'):
                            url = 'http://xiaorui.cc' + url
                            r = self.fetch_page(url)
                            url_ = self.upload_img(url.split('/')[-1], r.content, 'crawl/')
                            img_list.append(url_)
                            new_img.attrs['src'] = url_
                            i.replace_with(new_img)
                        else:

# Path: app/tasks/crawler.py
# class BaseCrawler(metaclass=ABCMeta):
#     def __init__(self):
#         self.site = None
#         self.timeout = 30
#         self.session = requests.Session()
#         self.headers = {
#             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/'
#                           '60.0.3112.113 Safari/537.36'
#         }
#         self.today = datetime.datetime.today().strftime('%Y-%m-%d')
# 
#     def fetch_page(self, url, data=None, headers=None, timeout=None):
#         try:
#             timeout = timeout or self.timeout
#             headers = headers or self.headers
#             if data is None:
#                 r = self.session.get(url=url, timeout=timeout, headers=headers)
#             else:
#                 r = self.session.post(url=url, data=data, timeout=timeout, headers=headers)
#         except Exception as e:
#             print(str(e))
#         else:
#             return r
# 
#     def login(self):
#         cookies = self._login()
#         self.session.cookies = cookies
# 
#     def _login(self):
#         pass
# 
#     def parser(self):
#         pass
# 
#     def save(self, title, body, style='转载', category='Python 进阶', post_img=choice_img()):
#         app = create_app('default')
#         with app.app_context():
#             post = Post(title=title,
#                         body=body,
#                         style=style,
#                         category=category,
#                         author_id=1,
#                         post_img=post_img,
#                         created=datetime.datetime.now(),
#                         source=self.site)
#             db.session.add(post)
#             db.session.commit()
# 
#     def is_today(self, s):
#         return s == datetime.datetime.now().strftime('%Y-%m-%d')
# 
#     def start(self):
#         pass
# 
#     def upload_img(self, file_name, file, prefix, domian_name='https://static.51qinqing.com'):
#         u = UploadToQiniu(file, prefix, mark=True)
#         ret, info = u.upload_web(prefix + file_name, file)
#         key = ret['key']
#         url = domian_name + '/' + key
#         return url
# 
#     @abstractmethod
#     def parse(self):
#         pass
# 
# Path: app/shares.py
# def choice_img():
#     return 'https://static.51qinqing.com/postimg/' + random.choice(img_list)


# Path: app/tasks/xiaorui.py
from .crawler import BaseCrawler
from lxml import etree
from bs4 import BeautifulSoup
from ..shares import choice_img
import random
import datetime

                    i.decompose()
                foot = soup.find('div', style="font-family:'Helvetica Neue';font-size:14px;")
                next_ = soup.find('div', clas="zan-page bs-example")
                copy_right = soup.find('div', class_="copyright alert alert-success")
                foot.decompose()
                next_.decompose()
                copy_right.decompose()
                as_ = soup.find_all('a')
                for i in as_:
                    i.decompose()
                tree = etree.HTML(r.text)
                title = tree.xpath('//div[@class="title-article"]/h1/a/text()')[0]
                print(title)
                codes = soup.find_all('div',
                                      class_='crayon-syntax crayon-theme-solarized-light crayon-font-monaco crayon-os-pc print-yes notranslate')
                images = soup.find('div', class_='centent-article').find_all('img')
                img_list = []
                if images:
                    for i in images:
                        new_img = soup.new_tag('img')
                        new_img.attrs['width'] = '100%'
                        url = i.get('src')
                        if str(url).startswith('/wp-content'):
                            url = 'http://xiaorui.cc' + url
                            r = self.fetch_page(url)
                            url_ = self.upload_img(url.split('/')[-1], r.content, 'crawl/')
                            img_list.append(url_)
                            new_img.attrs['src'] = url_
                            i.replace_with(new_img)
                        else:

                            new_img.attrs['src'] = choice_img()