<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
illustrator is used to find all images from a web page
"""
# @author chengdujin
# @contact <EMAIL>
# @created Jul. 23, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup
from config.settings import hparser
from config.settings import logger
from cStringIO import StringIO
from PIL import Image
import os
import random
import re
import requests
from slimmer import html_slimmer
import urllib2

# CONSTANTS
from config.settings import HEADERS
from config.settings import IMAGES_LOCAL_DIR
from config.settings import IMAGES_PUBLIC_DIR
from config.settings import MIN_IMAGE_SIZE
from config.settings import UCK_TIMEOUT

# creat images local directory if it does not exist
if not os.path.exists(IMAGES_LOCAL_DIR):
    os.mkdir(IMAGES_LOCAL_DIR)


class NormalizedImage:

    """
    Class of normalized image
    """

    def __init__(self, image_url=None, referer=None):
        if not image_url:
            logger.error('Method malformed!')
            raise Exception('Method malformed!')

        self._image_url, self._image_html = self._analyze(image_url, referer)
        self._image_size = self._calculate_size(self._image_html)

        self._clean_data()

    def _analyze(self, image_url=None, referer=None):
        """
        remove CDN prefix, if any; and read image data
        """
        if not image_url:
            logger.error('Method malformed!')
            raise Exception('Method malformed!')

        image_url = image_url.replace("\/", "/").strip()
        image_url = urllib2.unquote(hparser.unescape(image_url))

        # as the name could be http://xxx.com/yyy--http://zzz.jpg
        # or http://xxx.com/yyy--https://zzz.jpg
        last_http_index = image_url.rfind('http')
        image_url = image_url[last_http_index:]

        response = None
        if referer:
            HEADERS['Referer'] = referer
        try:
            response = requests.get(
                image_url, headers=HEADERS, timeout=UCK_TIMEOUT)
            # avoid redirected URL
            image_url = response.url
            # either exception or wrong HTTP code
            if response.status_code >= 400:
                raise Exception('Response code %s' % response.status_code)
        except Exception as k:
            logger.info('%s for %s' % (str(k), str(image_url)))
            try:
                # CDN URL could be formed as http:/xxxx.jpg
                path = re.split('https?://?', image_url)[-1]
                scheme = requests.utils.urlparse(image_url).scheme
                image_url = '%s://%s' % (scheme, path)

                response = requests.get(
                    image_url, headers=HEADERS, timeout=UCK_TIMEOUT)
                # avoid redirected URL
                image_url = response.url
                if response.status_code >= 400:
                    raise Exception('Response code %s' % response.status_code)
            except Exception as k:
                logger.error('%s for %s' % (str(k), str(image_url)))
                raise Exception('%s for %s' % (str(k), str(image_url)))

        if response and response.status_code < 400 and response.content:
            # GIF is not supported yet
            pr = requests.utils.urlparse(image_url)
            image_url_address = pr.netloc + pr.path
            #if image_url_address.lower().endswith('.gif'):
            #    raise Exception('GIF is not supported! %s' % str(image_url))
            #else:
            image_html = urllib2.unquote(hparser.unescape(response.content))
            image_url = self._check_image(image_url, image_html)
            return str(image_url), str(image_html)
        else:
            logger.error('Cannot parse %s' % str(image_url))
            raise Exception('Cannot parse %s' % str(image_url))

    def _calculate_size(self, image_html=None):
        """
        read data into memory and return the image size
        """
        if not image_html:
            logger.error('No data is found for [%s]!' % self._image_url)
            raise Exception('No data is found for [%s]!' % self._image_url)<commit_msg>Correct a bug<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
illustrator is used to find all images from a web page
"""
# @author chengdujin
# @contact <EMAIL>
# @created Jul. 23, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup
from config.settings import hparser
from config.settings import logger
from cStringIO import StringIO
from PIL import Image
import os
import random
import re
import requests
from slimmer import html_slimmer
import urllib2

# CONSTANTS
from config.settings import HEADERS
from config.settings import IMAGES_LOCAL_DIR
from config.settings import IMAGES_PUBLIC_DIR
from config.settings import MIN_IMAGE_SIZE
from config.settings import UCK_TIMEOUT

# creat images local directory if it does not exist
if not os.path.exists(IMAGES_LOCAL_DIR):
    os.mkdir(IMAGES_LOCAL_DIR)


class NormalizedImage:

    """
    Class of normalized image
    """

    def __init__(self, image_url=None, referer=None):
        if not image_url:
            logger.error('Method malformed!')
            raise Exception('Method malformed!')

        self._image_url, self._image_html = self._analyze(image_url, referer)
        self._image_size = self._calculate_size(self._image_html)

        self._clean_data()

    def _analyze(self, image_url=None, referer=None):
        """
        remove CDN prefix, if any; and read image data
        """
        if not image_url:
            logger.error('Method malformed!')
            raise Exception('Method malformed!')

        image_url = image_url.replace("\/", "/").strip()
        image_url = urllib2.unquote(hparser.unescape(image_url))

        # as the name could be http://xxx.com/yyy--http://zzz.jpg
        # or http://xxx.com/yyy--https://zzz.jpg
        last_http_index = image_url.rfind('http')
        image_url = image_url[last_http_index:]

        response = None
        if referer:
            HEADERS['Referer'] = referer
        try:
            response = requests.get(
                image_url, headers=HEADERS, timeout=UCK_TIMEOUT)
            # avoid redirected URL
            image_url = response.url
            # either exception or wrong HTTP code
            if response.status_code >= 400:
                raise Exception('Response code %s' % response.status_code)
        except Exception as k:
            logger.info('%s for %s' % (str(k), str(image_url)))
            try:
                # CDN URL could be formed as http:/xxxx.jpg
                path = re.split('https?://?', image_url)[-1]
                scheme = requests.utils.urlparse(image_url).scheme
                image_url = '%s://%s' % (scheme, path)

                response = requests.get(
                    image_url, headers=HEADERS, timeout=UCK_TIMEOUT)
                # avoid redirected URL
                image_url = response.url
                if response.status_code >= 400:
                    raise Exception('Response code %s' % response.status_code)
            except Exception as k:
                logger.error('%s for %s' % (str(k), str(image_url)))
                raise Exception('%s for %s' % (str(k), str(image_url)))

        if response and response.status_code < 400 and response.content:
            # GIF is not supported yet
            pr = requests.utils.urlparse(image_url)
            image_url_address = pr.netloc + pr.path
            #if image_url_address.lower().endswith('.gif'):
            #    raise Exception('GIF is not supported! %s' % str(image_url))
            #else:
            #image_html = urllib2.unquote(hparser.unescape(response.content))
            image_html = response.content
            image_url = self._check_image(image_url, image_html)
            return str(image_url), str(image_html)
        else:
            logger.error('Cannot parse %s' % str(image_url))
            raise Exception('Cannot parse %s' % str(image_url))

    def _calculate_size(self, image_html=None):
        """
        read data into memory and return the image size
        """
        if not image_html:
            logger.error('No data is found for [%s]!' % self._image_url)
            raise Exception('No data is found for [%s]!' % self._image_url)
{'fraction_non_alphanumeric': 0.10327694309712528, 'fraction_numerical': 0.004968650183366852, 'mean_word_length': 2.319199057714959, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1342552', 'n_tokens_mistral': 2510, 'n_tokens_neox': 2411, 'n_words': 651}