<commit_before>"""
This contains functions and classes used to evaluate if images are acceptable (do not show improper content, etc), and
to send them to S3.
"""

from PIL import Image
import urlparse
import requests
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from django.conf import settings
import pickle
import logging

log = logging.getLogger(__name__)

#Domains where any image linked to can be trusted to have acceptable content.
TRUSTED_IMAGE_DOMAINS = [
    'wikipedia.com',
    'wikipedia.net',
    'wikipedia.org',
    'edxuploads.s3.amazonaws.com'
]

#Suffixes that are allowed in image urls
ALLOWABLE_IMAGE_SUFFIXES = [
    'jpg',
    'png',
    'gif'
]

#Maximum allowed dimensions (x and y) for an uploaded image
MAX_ALLOWED_IMAGE_DIM = 1500

#Dimensions to which image is resized before it is evaluated for color count, etc
MAX_IMAGE_DIM = 150

#Maximum number of colors that should be counted in ImageProperties
MAX_COLORS_TO_COUNT = 16

#Maximum number of colors allowed in an uploaded image
MAX_COLORS = 20

class ImageProperties(object):
    """
    Class to check properties of an image and to validate if they are allowed.
    """
    def __init__(self, image):
        """
        Initializes class variables
        @param image: Image object (from PIL)
        @return: None
        """
        self.image = image
        image_size = self.image.size
        self.image_too_large = False
        if image_size[0] > MAX_ALLOWED_IMAGE_DIM or image_size[1] > MAX_ALLOWED_IMAGE_DIM:
            self.image_too_large = True
        if image_size[0] > MAX_IMAGE_DIM or image_size[1] > MAX_IMAGE_DIM:
            self.image = self.image.resize((MAX_IMAGE_DIM, MAX_IMAGE_DIM))
            self.image_size = self.image.size

    def count_colors(self):
        """
        Counts the number of colors in an image, and matches them to the max allowed
        @return: boolean true if color count is acceptable, false otherwise
        """
        colors = self.image.getcolors(MAX_COLORS_TO_COUNT)
        if colors is None:
            colors = MAX_COLORS_TO_COUNT
        else:
            colors = len(colors)

        too_many_colors = (colors <= MAX_COLORS)
        log.debug("Too many colors: {0}".format(too_many_colors))
        return too_many_colors

    def get_skin_ratio(self):
        """
        Gets the ratio of skin tone colors in an image
        @return: True if the ratio is low enough to be acceptable, false otherwise
        """
        im = self.image
        skin = sum([count for count, rgb in im.getcolors(im.size[0] * im.size[1]) if
                    rgb[0] > 60 and rgb[1] < (rgb[0] * 0.85) and rgb[2] < (rgb[0] * 0.7) and rgb[1] > (rgb[0] * 0.4) and
                    rgb[2] > (rgb[0] * 0.2)])
        bad_color_val = float(skin) / float(im.size[0] * im.size[1])
        if bad_color_val > .4:
            is_okay = False
        else:
            is_okay = True
        log.debug("Skin ratio okay: {0}".format(is_okay))
        return is_okay

    def run_tests(self):
        """
        Does all available checks on an image to ensure that it is okay (size, skin ratio, colors)
        @return: Boolean indicating whether or not image passes all checks
        """
        #image_is_okay = self.count_colors() and self.get_skin_ratio() and not self.image_too_large
        image_is_okay = self.count_colors() and not self.image_too_large
        log.debug("Image too large: {0}".format(self.image_too_large))
        log.debug("Image Okay: {0}".format(image_is_okay))
        return image_is_okay


class URLProperties(object):
    """
    Checks to see if a URL points to acceptable content.  Added to check if students are submitting reasonable
    links to the peer grading image functionality of the external grading service.
    """
    def __init__(self, url_string):
        self.url_string = url_string
<commit_msg>Fix var names, remove debug statements<commit_after>"""
This contains functions and classes used to evaluate if images are acceptable (do not show improper content, etc), and
to send them to S3.
"""

from PIL import Image
import urlparse
import requests
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from django.conf import settings
import pickle
import logging

log = logging.getLogger(__name__)

#Domains where any image linked to can be trusted to have acceptable content.
TRUSTED_IMAGE_DOMAINS = [
    'wikipedia.com',
    'wikipedia.net',
    'wikipedia.org',
    'edxuploads.s3.amazonaws.com'
]

#Suffixes that are allowed in image urls
ALLOWABLE_IMAGE_SUFFIXES = [
    'jpg',
    'png',
    'gif'
]

#Maximum allowed dimensions (x and y) for an uploaded image
MAX_ALLOWED_IMAGE_DIM = 1500

#Dimensions to which image is resized before it is evaluated for color count, etc
MAX_IMAGE_DIM = 150

#Maximum number of colors that should be counted in ImageProperties
MAX_COLORS_TO_COUNT = 16

#Maximum number of colors allowed in an uploaded image
MAX_COLORS = 20

class ImageProperties(object):
    """
    Class to check properties of an image and to validate if they are allowed.
    """
    def __init__(self, image):
        """
        Initializes class variables
        @param image: Image object (from PIL)
        @return: None
        """
        self.image = image
        image_size = self.image.size
        self.image_too_large = False
        if image_size[0] > MAX_ALLOWED_IMAGE_DIM or image_size[1] > MAX_ALLOWED_IMAGE_DIM:
            self.image_too_large = True
        if image_size[0] > MAX_IMAGE_DIM or image_size[1] > MAX_IMAGE_DIM:
            self.image = self.image.resize((MAX_IMAGE_DIM, MAX_IMAGE_DIM))
            self.image_size = self.image.size

    def count_colors(self):
        """
        Counts the number of colors in an image, and matches them to the max allowed
        @return: boolean true if color count is acceptable, false otherwise
        """
        colors = self.image.getcolors(MAX_COLORS_TO_COUNT)
        if colors is None:
            color_count = MAX_COLORS_TO_COUNT
        else:
            color_count = len(colors)

        too_many_colors = (color_count <= MAX_COLORS)
        return too_many_colors

    def get_skin_ratio(self):
        """
        Gets the ratio of skin tone colors in an image
        @return: True if the ratio is low enough to be acceptable, false otherwise
        """
        im = self.image
        skin = sum([count for count, rgb in im.getcolors(im.size[0] * im.size[1]) if
                    rgb[0] > 60 and rgb[1] < (rgb[0] * 0.85) and rgb[2] < (rgb[0] * 0.7) and rgb[1] > (rgb[0] * 0.4) and
                    rgb[2] > (rgb[0] * 0.2)])
        bad_color_val = float(skin) / float(im.size[0] * im.size[1])
        if bad_color_val > .4:
            is_okay = False
        else:
            is_okay = True
        return is_okay

    def run_tests(self):
        """
        Does all available checks on an image to ensure that it is okay (size, skin ratio, colors)
        @return: Boolean indicating whether or not image passes all checks
        """
        #image_is_okay = self.count_colors() and self.get_skin_ratio() and not self.image_too_large
        image_is_okay = self.count_colors() and not self.image_too_large
        return image_is_okay


class URLProperties(object):
    """
    Checks to see if a URL points to acceptable content.  Added to check if students are submitting reasonable
    links to the peer grading image functionality of the external grading service.
    """
    def __init__(self, url_string):
        self.url_string = url_string

{'fraction_non_alphanumeric': 0.0832235975770345, 'fraction_numerical': 0.012378193310508295, 'mean_word_length': 2.6940661478599224, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4936890', 'n_tokens_mistral': 2353, 'n_tokens_neox': 2272, 'n_words': 774}