<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals, division

import warnings
import io
from psd_tools.utils import be_array_from_bytes
from psd_tools.constants import (
    Compression, ChannelID, ColorMode, ImageResourceID)
from psd_tools import icc_profiles

try:
    from PIL import Image, ImageDraw, ImageMath
    if hasattr(Image, 'frombytes'):
        frombytes = Image.frombytes
    else:
        frombytes = Image.fromstring  # PIL and older Pillow versions
except ImportError:
    Image = None
    ImageDraw = None

try:
    from PIL import ImageCms
except ImportError:
    ImageCms = None


def tobytes(image):
    # Some versions of PIL are missing the tobytes alias for tostring
    if hasattr(image, 'tobytes'):
        return image.tobytes()
    else:
        return image.tostring()


def extract_layer_image(decoded_data, layer_index):
    """
    Converts a layer from the ``decoded_data`` to a PIL image.
    """
    layers = decoded_data.layer_and_mask_data.layers
    layer = layers.layer_records[layer_index]

    return _channel_data_to_PIL(
        channel_data=layers.channel_image_data[layer_index],
        channel_ids=_get_layer_channel_ids(layer),
        color_mode=decoded_data.header.color_mode,  # XXX?
        size=(layer.width(), layer.height()),
        depth=decoded_data.header.depth,
        icc_profile=get_icc_profile(decoded_data)
    )


def extract_layer_mask(decoded_data, layer_index, real_mask):
    """
    Converts a layer mask from the ``decoded_data`` to a PIL image.

    If ``real_mask`` is True, extract real mask consisting of both bitmap and
    vector mask.
    """
    layers = decoded_data.layer_and_mask_data.layers
    layer = layers.layer_records[layer_index]
    mask_data = layer.mask_data
    if not mask_data:
        return None
    real_mask = real_mask and mask_data.real_flags
    if real_mask:
        size = (mask_data.real_right - mask_data.real_left,
                mask_data.real_bottom - mask_data.real_top)
    else:
        size = (mask_data.right - mask_data.left,
                mask_data.bottom - mask_data.top)

    return _mask_data_to_PIL(
        channel_data=layers.channel_image_data[layer_index],
        channel_ids=_get_layer_channel_ids(layer),
        size=size,
        depth=decoded_data.header.depth,
        real_mask=real_mask
    )


def extract_composite_image(decoded_data):
    """
    Converts a composite (merged) image from the ``decoded_data``
    to a PIL image.
    """
    header = decoded_data.header
    size = header.width, header.height
    if size == (0, 0):
        return

    channel_ids = _get_header_channel_ids(header)
    if channel_ids is None:
        warnings.warn(
            "This number of channels (%d) is unsupported for this color mode"
            "(%s)" % (header.number_of_channels, header.color_mode))
        return
    if len(channel_ids) > len(decoded_data.image_data):
        warnings.warn("Image data is broken")
        return

    image = _channel_data_to_PIL(
        channel_data=decoded_data.image_data,
        channel_ids=channel_ids,
        color_mode=header.color_mode,
        size=size,
        depth=header.depth,
        icc_profile=get_icc_profile(decoded_data)
    )

    # Composed image is blended into white background. Remove here.
    return _remove_white_background(image)


def get_icc_profile(decoded_data):
    """
    Return ICC image profile if it exists and was correctly decoded
    """
    # fixme: move this function somewhere?
    icc_profiles = [res.data for res in decoded_data.image_resource_blocks
                    if res.resource_id == ImageResourceID.ICC_PROFILE]

    if not icc_profiles:
        return None

    icc_profile = icc_profiles[0]

    if isinstance(icc_profile, bytes):  # profile was not decoded
        return None

    return icc_profile


def apply_opacity(im, opacity):
    """ Apply opacity to an image. """
    if im.mode in ('RGBA', 'LA'):
        channels = list(im.split())
        opacity_scale = opacity / 255.
        channels[-1] = channels[-1].point(lambda i: int(i * opacity_scale))
        return Image.merge(im.mode, channels)
    elif im.mode in ('RGB', 'L'):
        im.putalpha(opacity)
        return im
    else:
        warnings.warn("%s converted to RGB" % im.mode)
        im = im.convert('RGB')
        im.putalpha(opacity)
        return im


def pattern_to_PIL(pattern):
    channels = [_decompress_pattern_channel(c) for c in pattern.data.channels]
    if not all(channels):
        return None

    image = None
    if len(channels) == 1:
        image = channels[0]
    elif len(channels) == 3:
        image = Image.merge('RGB', channels)
    elif len(channels) == 4:
        image = Image.merge('RGBA', channels)
    return image


def draw_polygon(bbox, anchors, fill=(255, 255, 255, 255)):
    image = Image.new("RGBA", (bbox.width, bbox.height),
                      color=(255, 255, 255, 0))<commit_msg>Support CMYK pattern to PIL conversion<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals, division

import warnings
import io
from psd_tools.utils import be_array_from_bytes
from psd_tools.constants import (
    Compression, ChannelID, ColorMode, ImageResourceID)
from psd_tools import icc_profiles

try:
    from PIL import Image, ImageDraw, ImageMath
    if hasattr(Image, 'frombytes'):
        frombytes = Image.frombytes
    else:
        frombytes = Image.fromstring  # PIL and older Pillow versions
except ImportError:
    Image = None
    ImageDraw = None

try:
    from PIL import ImageCms
except ImportError:
    ImageCms = None


def tobytes(image):
    # Some versions of PIL are missing the tobytes alias for tostring
    if hasattr(image, 'tobytes'):
        return image.tobytes()
    else:
        return image.tostring()


def extract_layer_image(decoded_data, layer_index):
    """
    Converts a layer from the ``decoded_data`` to a PIL image.
    """
    layers = decoded_data.layer_and_mask_data.layers
    layer = layers.layer_records[layer_index]

    return _channel_data_to_PIL(
        channel_data=layers.channel_image_data[layer_index],
        channel_ids=_get_layer_channel_ids(layer),
        color_mode=decoded_data.header.color_mode,  # XXX?
        size=(layer.width(), layer.height()),
        depth=decoded_data.header.depth,
        icc_profile=get_icc_profile(decoded_data)
    )


def extract_layer_mask(decoded_data, layer_index, real_mask):
    """
    Converts a layer mask from the ``decoded_data`` to a PIL image.

    If ``real_mask`` is True, extract real mask consisting of both bitmap and
    vector mask.
    """
    layers = decoded_data.layer_and_mask_data.layers
    layer = layers.layer_records[layer_index]
    mask_data = layer.mask_data
    if not mask_data:
        return None
    real_mask = real_mask and mask_data.real_flags
    if real_mask:
        size = (mask_data.real_right - mask_data.real_left,
                mask_data.real_bottom - mask_data.real_top)
    else:
        size = (mask_data.right - mask_data.left,
                mask_data.bottom - mask_data.top)

    return _mask_data_to_PIL(
        channel_data=layers.channel_image_data[layer_index],
        channel_ids=_get_layer_channel_ids(layer),
        size=size,
        depth=decoded_data.header.depth,
        real_mask=real_mask
    )


def extract_composite_image(decoded_data):
    """
    Converts a composite (merged) image from the ``decoded_data``
    to a PIL image.
    """
    header = decoded_data.header
    size = header.width, header.height
    if size == (0, 0):
        return

    channel_ids = _get_header_channel_ids(header)
    if channel_ids is None:
        warnings.warn(
            "This number of channels (%d) is unsupported for this color mode"
            "(%s)" % (header.number_of_channels, header.color_mode))
        return
    if len(channel_ids) > len(decoded_data.image_data):
        warnings.warn("Image data is broken")
        return

    image = _channel_data_to_PIL(
        channel_data=decoded_data.image_data,
        channel_ids=channel_ids,
        color_mode=header.color_mode,
        size=size,
        depth=header.depth,
        icc_profile=get_icc_profile(decoded_data)
    )

    # Composed image is blended into white background. Remove here.
    return _remove_white_background(image)


def get_icc_profile(decoded_data):
    """
    Return ICC image profile if it exists and was correctly decoded
    """
    # fixme: move this function somewhere?
    icc_profiles = [res.data for res in decoded_data.image_resource_blocks
                    if res.resource_id == ImageResourceID.ICC_PROFILE]

    if not icc_profiles:
        return None

    icc_profile = icc_profiles[0]

    if isinstance(icc_profile, bytes):  # profile was not decoded
        return None

    return icc_profile


def apply_opacity(im, opacity):
    """ Apply opacity to an image. """
    if im.mode in ('RGBA', 'LA'):
        channels = list(im.split())
        opacity_scale = opacity / 255.
        channels[-1] = channels[-1].point(lambda i: int(i * opacity_scale))
        return Image.merge(im.mode, channels)
    elif im.mode in ('RGB', 'L'):
        im.putalpha(opacity)
        return im
    else:
        warnings.warn("%s converted to RGB" % im.mode)
        im = im.convert('RGB')
        im.putalpha(opacity)
        return im


def pattern_to_PIL(pattern):
    channels = [_decompress_pattern_channel(c) for c in pattern.data.channels]
    if not all(channels):
        warnings.warn("Failed to decompress pattern")
        return None
    # Bitmap = 0; Grayscale = 1; Indexed = 2; RGB = 3; CMYK = 4;
    # Multichannel = 7; Duotone = 8; Lab = 9
    image_mode = pattern.image_mode
    image = None
    if image_mode == 1:
        assert len(channels) in (1, 2)
        image = channels[0]
        if len(channels) > 1:
            image.putalpha(channels[1])
    elif image_mode == 3:
        assert len(channels) in (3, 4)
        image = Image.merge('RGB', channels[:3])
        if len(channels) > 3:
            image.putalpha(channels[3])
    elif image_mode == 4:
        assert len(channels) in (4, 5)
        image = Image.merge('CMYK', channels[:4]).convert('RGB')
        if len(channels) > 4:
            image.putalpha(channels[4])
    else:
        warnings.warn("Unsupported image mode %d" % pattern.image_mode)
        if len(channels) > 0:
            image = channels[0]
    return image


def draw_polygon(bbox, anchors, fill=(255, 255, 255, 255)):
    image = Image.new("RGBA", (bbox.width, bbox.height),
                      color=(255, 255, 255, 0))
{'fraction_non_alphanumeric': 0.09448450229422231, 'fraction_numerical': 0.008802322314823485, 'mean_word_length': 2.7926136363636362, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3816175', 'n_tokens_mistral': 3427, 'n_tokens_neox': 3304, 'n_words': 838}