# Path: odpw/utils/error_handling.py
# def getExceptionString(e):
#     try:
#         if isinstance(e,ckanapi.errors.CKANAPIError):
#             try:
#                 err = literal_eval(e.extra_msg)
#                 return str(type(e))+":"+str(err[2])
#             except Exception:
#                 return str(type(e))+":"+str(e.extra_msg)
#         else:
#             if e.message:
#                 return str(type(e))+":"+str(e.message)
#             if e.message:
#                 return str(type(e))+":"
#     except Exception as e:
#         log.error("Get Exception string", exctype=type(e), excmsg=e.message,exc_info=True)
#         return 601
# 
# class ErrorHandler():
# 
#     exceptions=defaultdict(long)
# 
#     DEBUG=False
# 
#     @classmethod
#     def handleError(cls, log, msg=None, exception=None, debug=False, **kwargs):
#         name=type(exception).__name__
#         cls.exceptions[name] +=1
# 
#         if debug:
#             print(traceback.format_exc())
# 
#         log.error(msg, exctype=type(exception), excmsg=exception.message, **kwargs)
# 
#     @classmethod
#     def printStats(cls):
#         print '>>>','--*'*10,'EXCEPTIONS','*--'*10
#         if len(cls.exceptions)==0:
#             print "No exceptions handled"
#         else:
#             print "  Numbers of Exceptions:"
#             for exc, count in cls.exceptions.iteritems():
#                 print " ",exc, count
#         print '<<<','--*'*25
# 
# Path: odpw/core/model.py
# class ResourceInfo(Base):
#     __tablename__ = tab_resourcesinfo
# 
#     uri= Column(String, primary_key=True)
#     snapshot= Column(SmallInteger, primary_key=True)
#     timestamp= Column(TIMESTAMP)
#     status=Column(SmallInteger)
#     exc=Column(String)
#     header=Column(JSONB)
#     mime=Column(String)
#     size=Column(BigInteger)
# 
# class ResourceCrawlLog(Base):
#     __tablename__ = tab_resourcescrawllog
# 
#     uri= Column(String, primary_key=True)
#     snapshot= Column(SmallInteger, primary_key=True)
#     timestamp= Column(TIMESTAMP, primary_key=True)
#     status=Column(SmallInteger, index=True)
# 
#     exc=Column(String)
#     header=Column(JSONB)
#     mime=Column(String)
#     size=Column(BigInteger)
#     crawltime=Column(BigInteger)
# 
#     referrer=Column( String)
#     disklocation=Column( String)
#     digest=Column( String)
#     contentchanged=Column( Integer)
#     domain=Column( String, index=True)


import datetime
import gzip
import hashlib
import os
import time
import urllib
import structlog
import sys
import twisted
import scrapy
from urlparse import urlparse
from scrapy.http import Response
from odpw.utils.error_handling import getExceptionString, ErrorHandler
from odpw.core.model import ResourceInfo, ResourceCrawlLog
                    domain = '{uri.netloc}'.format(uri=parsed_uri)
                except:
                    domain='error'
                request.meta['domain']=domain
            #create folder and file
            domain=request.meta['domain']

            #compute digest and filesize
            digest=hashlib.md5(content).hexdigest()
            request.meta['digest']=digest
            request.meta['size']=sys.getsizeof(content)

            #check if digest exists?, if yes, get file location and file size
            last_digest= spider.api.getLastDigest(uri=response.url)

            request.meta['contentchanged']=0 if last_digest and last_digest[0]==digest else 1

            try:
                filename=request.meta['git']
                if not os.path.exists(os.path.dirname(filename)):
                    os.makedirs(os.path.dirname(filename))

                with open(filename,'wb') as fw:
                    try:
                        fw.write(content)
                    except Exception as e:
                        request.meta['error']=getExceptionString(e)
                        ErrorHandler.handleError("Writing file", exception=e)
            except Exception as e:
                ErrorHandler.handleError(log,'file_download',exception=e, uri=request.url)

# Path: odpw/utils/error_handling.py
# def getExceptionString(e):
#     try:
#         if isinstance(e,ckanapi.errors.CKANAPIError):
#             try:
#                 err = literal_eval(e.extra_msg)
#                 return str(type(e))+":"+str(err[2])
#             except Exception:
#                 return str(type(e))+":"+str(e.extra_msg)
#         else:
#             if e.message:
#                 return str(type(e))+":"+str(e.message)
#             if e.message:
#                 return str(type(e))+":"
#     except Exception as e:
#         log.error("Get Exception string", exctype=type(e), excmsg=e.message,exc_info=True)
#         return 601
# 
# class ErrorHandler():
# 
#     exceptions=defaultdict(long)
# 
#     DEBUG=False
# 
#     @classmethod
#     def handleError(cls, log, msg=None, exception=None, debug=False, **kwargs):
#         name=type(exception).__name__
#         cls.exceptions[name] +=1
# 
#         if debug:
#             print(traceback.format_exc())
# 
#         log.error(msg, exctype=type(exception), excmsg=exception.message, **kwargs)
# 
#     @classmethod
#     def printStats(cls):
#         print '>>>','--*'*10,'EXCEPTIONS','*--'*10
#         if len(cls.exceptions)==0:
#             print "No exceptions handled"
#         else:
#             print "  Numbers of Exceptions:"
#             for exc, count in cls.exceptions.iteritems():
#                 print " ",exc, count
#         print '<<<','--*'*25
# 
# Path: odpw/core/model.py
# class ResourceInfo(Base):
#     __tablename__ = tab_resourcesinfo
# 
#     uri= Column(String, primary_key=True)
#     snapshot= Column(SmallInteger, primary_key=True)
#     timestamp= Column(TIMESTAMP)
#     status=Column(SmallInteger)
#     exc=Column(String)
#     header=Column(JSONB)
#     mime=Column(String)
#     size=Column(BigInteger)
# 
# class ResourceCrawlLog(Base):
#     __tablename__ = tab_resourcescrawllog
# 
#     uri= Column(String, primary_key=True)
#     snapshot= Column(SmallInteger, primary_key=True)
#     timestamp= Column(TIMESTAMP, primary_key=True)
#     status=Column(SmallInteger, index=True)
# 
#     exc=Column(String)
#     header=Column(JSONB)
#     mime=Column(String)
#     size=Column(BigInteger)
#     crawltime=Column(BigInteger)
# 
#     referrer=Column( String)
#     disklocation=Column( String)
#     digest=Column( String)
#     contentchanged=Column( Integer)
#     domain=Column( String, index=True)


# Path: odpw/core/scrapy_middlewares.py
import datetime
import gzip
import hashlib
import os
import time
import urllib
import structlog
import sys
import twisted
import scrapy
from urlparse import urlparse
from scrapy.http import Response
from odpw.utils.error_handling import getExceptionString, ErrorHandler
from odpw.core.model import ResourceInfo, ResourceCrawlLog

                    domain = '{uri.netloc}'.format(uri=parsed_uri)
                except:
                    domain='error'
                request.meta['domain']=domain
            #create folder and file
            domain=request.meta['domain']

            #compute digest and filesize
            digest=hashlib.md5(content).hexdigest()
            request.meta['digest']=digest
            request.meta['size']=sys.getsizeof(content)

            #check if digest exists?, if yes, get file location and file size
            last_digest= spider.api.getLastDigest(uri=response.url)

            request.meta['contentchanged']=0 if last_digest and last_digest[0]==digest else 1

            try:
                filename=request.meta['git']
                if not os.path.exists(os.path.dirname(filename)):
                    os.makedirs(os.path.dirname(filename))

                with open(filename,'wb') as fw:
                    try:
                        fw.write(content)
                    except Exception as e:
                        request.meta['error']=getExceptionString(e)
                        ErrorHandler.handleError("Writing file", exception=e)
            except Exception as e:
                ErrorHandler.handleError(log,'file_download',exception=e, uri=request.url)

                request.meta['error']=getExceptionString(e)