def is_valid_response(type_whitelist, content_type_header):
    for type_regex in type_whitelist:
        if re.search(type_regex, content_type_header):
            return True
return False

def process_response(self, request, response, spider):
    """
        Only allow HTTP response types that that match the given list of 
        filtering regexs
        """
type_whitelist = getattr(spider, 'response_type_whitelist', None)
content_type_header = response.headers.get('content-type', None)
Python Scrapy - mimetype based filter to avoid non-text file downloads