# Path: scrapywork/models.py
# class ArticleRule():
# class CrawlRule(ArticleRule):
# class BaseRule(ArticleRule):
# class SpiderRulesSingletion(object):
# class ItemParser(object):
# class Mail(object):
# class DatabaseSingleton(object):
# class IPProxy(object):
#     def __init__(self):
#     def _parsexml(self,xmlpath):
#     def _parseCrawlrules(self,rule):
#     def _parseBaserules(self,rule):
#     def __init__(self,rule,response):
#     def parse_item(self):
#     def _fixhtml(self,html):
#     def _formatetime(self,time):
#     def __init__(self,sender,passwd,receiver):
#     def sendmail(self,subject,contents):
#     def get_maile_content(self):
#     def get_allitem_count(self):
#     def __init__(self):
#     def __del__(self):
#     def ensuretableexist(self):
#     def insertItem(self,url,site,title,time,type,publish,html,text,xml,sources):
#     def insertItem(self,item,source):
#     def queryItem(self,url):
#     def queryItemCount(self,site=None):
#     def __init__(self):
#     def _initipproxy(self,types,inlandcount,foreigncount):
#     def getinlandipproxy(self):
#     def getforeignipproxy(self):


from scrapy.exceptions import DropItem
from scrapywork.models import mysqldb
from urllib.request import urlretrieve
from scrapy.pipelines.images import ImagesPipeline
from PIL import Image
from io import BytesIO
from scrapy.pipelines.files import FileException
import scrapy
import six
import logging
import tempfile
import cv2
            return item
        except:
            raise DropItem('Cannot Insert %s into mysql database'% item)

            logging.info('db error')

    def _parse_sourceandhtml(self,images,files,html):
        source=[]
        htmlreplace=html
        for image in images:
            logging.info(image)
            imagename = image['path'].split('/')[-1]
            htmlreplace = htmlreplace.replace(str(image['url']), str(imagename))
            source.append(imagename)
            #hfs.UploadStreamXImages(imagename)
        for file in files:
            logging.info(file)
            filename = file['path'].split('/')[-1]
            htmlreplace = htmlreplace.replace(str(file['url']),str(filename))
            source.append(filename)
            #hfs.UploadStreamXFiles(filename)
        return '|'.join(source),htmlreplace

    def _mysqldb_insertitem(self,item):
        sources, html = self._parse_sourceandhtml(item['images'], item['files'], item['html'])
        time = item['time']
        item['html']=html

        # mysqldb.insertItem(url=item['url'],site=item['site'],title=item['title'],time=time,type=item['type'],publish=item['publish'],
        #                    html=html,text=item['text'],xml=item['xml'],sources=sources)

# Path: scrapywork/models.py
# class ArticleRule():
# class CrawlRule(ArticleRule):
# class BaseRule(ArticleRule):
# class SpiderRulesSingletion(object):
# class ItemParser(object):
# class Mail(object):
# class DatabaseSingleton(object):
# class IPProxy(object):
#     def __init__(self):
#     def _parsexml(self,xmlpath):
#     def _parseCrawlrules(self,rule):
#     def _parseBaserules(self,rule):
#     def __init__(self,rule,response):
#     def parse_item(self):
#     def _fixhtml(self,html):
#     def _formatetime(self,time):
#     def __init__(self,sender,passwd,receiver):
#     def sendmail(self,subject,contents):
#     def get_maile_content(self):
#     def get_allitem_count(self):
#     def __init__(self):
#     def __del__(self):
#     def ensuretableexist(self):
#     def insertItem(self,url,site,title,time,type,publish,html,text,xml,sources):
#     def insertItem(self,item,source):
#     def queryItem(self,url):
#     def queryItemCount(self,site=None):
#     def __init__(self):
#     def _initipproxy(self,types,inlandcount,foreigncount):
#     def getinlandipproxy(self):
#     def getforeignipproxy(self):


# Path: scrapywork/pipelines.py
from scrapy.exceptions import DropItem
from scrapywork.models import mysqldb
from urllib.request import urlretrieve
from scrapy.pipelines.images import ImagesPipeline
from PIL import Image
from io import BytesIO
from scrapy.pipelines.files import FileException
import scrapy
import six
import logging
import tempfile
import cv2

            return item
        except:
            raise DropItem('Cannot Insert %s into mysql database'% item)

            logging.info('db error')

    def _parse_sourceandhtml(self,images,files,html):
        source=[]
        htmlreplace=html
        for image in images:
            logging.info(image)
            imagename = image['path'].split('/')[-1]
            htmlreplace = htmlreplace.replace(str(image['url']), str(imagename))
            source.append(imagename)
            #hfs.UploadStreamXImages(imagename)
        for file in files:
            logging.info(file)
            filename = file['path'].split('/')[-1]
            htmlreplace = htmlreplace.replace(str(file['url']),str(filename))
            source.append(filename)
            #hfs.UploadStreamXFiles(filename)
        return '|'.join(source),htmlreplace

    def _mysqldb_insertitem(self,item):
        sources, html = self._parse_sourceandhtml(item['images'], item['files'], item['html'])
        time = item['time']
        item['html']=html

        # mysqldb.insertItem(url=item['url'],site=item['site'],title=item['title'],time=time,type=item['type'],publish=item['publish'],
        #                    html=html,text=item['text'],xml=item['xml'],sources=sources)

        mysqldb.insertItem(item,sources)