<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
image_scraper is used to find all images from a web page
"""
# @author chengdujin
# @contact <EMAIL>
# @created Jul. 23, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup
from config.settings import hparser
from config.settings import logger
from cStringIO import StringIO
import httplib
import Image
import os
import re
import thumbnail
import urllib2
import urlparse

# CONSTANTS
from config.settings import IMAGES_LOCAL_DIR
from config.settings import IMAGES_PUBLIC_DIR
from config.settings import MIN_IMAGE_SIZE
from config.settings import TRANSCODED_LOCAL_DIR
from config.settings import UCK_TIMEOUT

# creat images local directory if it does not exist
if not os.path.exists(IMAGES_LOCAL_DIR):
    os.mkdir(IMAGES_LOCAL_DIR)


def _check_image(image):
    """
    check an image if it matches with MIN_IMAGE_SIZE
    """
    if not image:
        logger.error('Method malformed!')
        return None

    try:
        if isinstance(image, dict) and 'url' in image:
            image_url = image['url']
            if _is_valid_image(image_url):
                width, height = thumbnail.get_image_size(image_url)
                image = {'url':image['url'], 'width':width, 'height':height}
                return image
        else:
            if _is_valid_image(image):
                width, height = thumbnail.get_image_size(image)
                if width and height:
                    return {'url': image, 'width': width, 'height': height}
                else:
                    logger.info('Cannot get image size')
                    return None
        return None
    except IOError as k:
        logger.error(str(k))
        return None


def dedupe_images(images):
    """
    remove same images
    image: {'url':xxx, 'width':yyy, 'height':zzz}
    images = [image, image, image]
    """
    if not images:
        return None

    image_urls = []

    def _exists(image):
        """
        return boolean if image exists in list image_urls
        """
        exists = image['url'] in image_urls
        if not exists:
            image_urls.append(image['url'])
            return False
        else:
            return True

    try:
        return filter(lambda x: not _exists(x), images)
    except Exception as k:
        logger.error(str(k))
        return None


def find_image(link=None):
    """
    find an image from the link
    """
    if not link:
        return None

    try:
        link_clean = _link_process(link)
        if link_clean:
            image_normalized = normalize(link_clean)
            return image_normalized[0] if image_normalized else None
        else:
            logger.info('Cannot parse [clean %s] [orginal %s] correctly' % (link_clean, link))
            return None
    except Exception as k:
        logger.error(str(k))
        return None


def find_images(content=None):
    """
    find out all images from content and its size info
    """
    if not content:
        return None

    try:
        # determine the type of content
        if isinstance(content, str) and content.startswith(TRANSCODED_LOCAL_DIR):
            # then its a file
            f = open(content, 'r')
            content = f.read()

        soup = BeautifulSoup(content.decode('utf-8', 'ignore'))
        images_normalized = []
        images = soup.findAll('img')

        for image in images:
            if image.get('src'):
                image_normalized = find_image(image.get('src'))
                if image_normalized:
                    images_normalized.append(image_normalized)

        return images_normalized
    except Exception as k:
        logger.error(str(k))
        return None


def find_biggest_image(images=None):
    """
    find the biggest in resolution from a pile of images
    """
    if not images:
        return None

    try:
        biggest = None
        for image in images:
            resolution_max = MIN_IMAGE_SIZE[0] * MIN_IMAGE_SIZE[1]
            if 'width' in image and 'height' in image:
                resolution_image = int(image['width']) * int(image['height'])
                if resolution_image > resolution_max:
                    biggest = image
                    resolution_max = resolution_image
            else:
                logger.error('Image malformed! %s' % str(image))
        return biggest
    except Exception as k:
        logger.error(str(k))
        return None


def _is_valid_image(image_url):
    """
    find out if the image has a resolution larger than MIN_IMAGE_SIZE
    """
    if not image_url:
        logger.error('Method malformed! URL [%s] is incorrect' % image_url)
        return False

    try:
        if _url_image_exists(image_url)
            # to avoid line length limit
            return True if image_pil.size[0] * image_pil.size[1] > MIN_IMAGE_SIZE[0] * MIN_IMAGE_SIZE[1] else False
        else:
            logger.info('%s is not an image' % image_url)
            return False
    except Exception as k:
        logger.error('%s' % str(k))
        return False

<commit_msg>Add get_image_size code from thumbnail.py to image_helper.py<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
image_scraper is used to find all images from a web page
"""
# @author chengdujin
# @contact <EMAIL>
# @created Jul. 23, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup
from config.settings import hparser
from config.settings import logger
from cStringIO import StringIO
import httplib
import Image
import os
import re
import urllib2
import urlparse

# CONSTANTS
from config.settings import IMAGES_LOCAL_DIR
from config.settings import IMAGES_PUBLIC_DIR
from config.settings import MIN_IMAGE_SIZE
from config.settings import TRANSCODED_LOCAL_DIR
from config.settings import UCK_TIMEOUT

# creat images local directory if it does not exist
if not os.path.exists(IMAGES_LOCAL_DIR):
    os.mkdir(IMAGES_LOCAL_DIR)


def _check_image(image):
    """
    check an image if it matches with MIN_IMAGE_SIZE
    """
    if not image:
        logger.error('Method malformed!')
        return None

    try:
        if isinstance(image, dict) and 'url' in image:
            image_url = image['url']
            if _is_valid_image(image_url):
                width, height = get_image_size(image_url)
                image = {'url':image['url'], 'width':width, 'height':height}
                return image
        else:
            if _is_valid_image(image):
                width, height = get_image_size(image)
                if width and height:
                    return {'url': image, 'width': width, 'height': height}
                else:
                    logger.info('Cannot get image size')
                    return None
        return None
    except IOError as k:
        logger.error(str(k))
        return None


def dedupe_images(images):
    """
    remove same images
    image: {'url':xxx, 'width':yyy, 'height':zzz}
    images = [image, image, image]
    """
    if not images:
        return None

    image_urls = []

    def _exists(image):
        """
        return boolean if image exists in list image_urls
        """
        exists = image['url'] in image_urls
        if not exists:
            image_urls.append(image['url'])
            return False
        else:
            return True

    try:
        return filter(lambda x: not _exists(x), images)
    except Exception as k:
        logger.error(str(k))
        return None


def find_image(link=None):
    """
    find an image from the link
    """
    if not link:
        return None

    try:
        link_clean = _link_process(link)
        if link_clean:
            image_normalized = normalize(link_clean)
            return image_normalized[0] if image_normalized else None
        else:
            logger.info('Cannot parse [clean %s] [orginal %s] correctly' % (link_clean, link))
            return None
    except Exception as k:
        logger.error(str(k))
        return None


def find_images(content=None):
    """
    find out all images from content and its size info
    """
    if not content:
        return None

    try:
        # determine the type of content
        if isinstance(content, str) and content.startswith(TRANSCODED_LOCAL_DIR):
            # then its a file
            f = open(content, 'r')
            content = f.read()

        soup = BeautifulSoup(content.decode('utf-8', 'ignore'))
        images_normalized = []
        images = soup.findAll('img')

        for image in images:
            if image.get('src'):
                image_normalized = find_image(image.get('src'))
                if image_normalized:
                    images_normalized.append(image_normalized)

        return images_normalized
    except Exception as k:
        logger.error(str(k))
        return None


def find_biggest_image(images=None):
    """
    find the biggest in resolution from a pile of images
    """
    if not images:
        return None

    try:
        biggest = None
        for image in images:
            resolution_max = MIN_IMAGE_SIZE[0] * MIN_IMAGE_SIZE[1]
            if 'width' in image and 'height' in image:
                resolution_image = int(image['width']) * int(image['height'])
                if resolution_image > resolution_max:
                    biggest = image
                    resolution_max = resolution_image
            else:
                logger.error('Image malformed! %s' % str(image))
        return biggest
    except Exception as k:
        logger.error(str(k))
        return None


def _is_valid_image(image_url):
    """
    find out if the image has a resolution larger than MIN_IMAGE_SIZE
    """
    if not image_url:
        logger.error('Method malformed! URL [%s] is incorrect' % image_url)
        return False

    try:
        if _url_image_exists(image_url)
            # to avoid line length limit
            return True if image_pil.size[0] * image_pil.size[1] > MIN_IMAGE_SIZE[0] * MIN_IMAGE_SIZE[1] else False
        else:
            logger.info('%s is not an image' % image_url)
            return False
    except Exception as k:
        logger.error('%s' % str(k))
        return False


def get_image_size(image_url):
    """
    docs needed
    """
    if not image_url:
        logger.error('Method malformed!')
        return None, None

    try:
        image_web = None
        if isinstance(image_url, str) or isinstance(image_url, unicode):
            logger.info('opening %s' % image_url)
            image_web = StringIO(
                urllib2.urlopen(image_url, timeout=UCK_TIMEOUT).read())
        else:
            logger.info('image_url is data')
            image_web = image_url

        if image_web:
            im = Image.open(image_web)
            width, height = im.size
            return width, height
        else:
            return None, None
    except Exception as k:
        logger.info('Problem:[%s] Source:[%s]' % (str(k), image_url))
        return None, None


{'fraction_non_alphanumeric': 0.08941997851772288, 'fraction_numerical': 0.003132832080200501, 'mean_word_length': 2.0993065187239943, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4808659', 'n_tokens_mistral': 3233, 'n_tokens_neox': 3142, 'n_words': 953}