<commit_before>#
#  The OpenDiamond Platform for Interactive Search
#
#  Copyright (c) 2011 Carnegie Mellon University
#  All rights reserved.
#
#  This software is distributed under the terms of the Eclipse Public
#  License, Version 1.0 which can be found in the file named LICENSE.
#  ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS SOFTWARE CONSTITUTES
#  RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT
#

from cStringIO import StringIO
import PIL.Image
import struct

class _AttributeCodec(object):
    '''Base class for a Diamond attribute codec.'''

    def encode(self, item):
        raise NotImplementedError()

    def decode(self, data):
        raise NotImplementedError()


class StringAttributeCodec(_AttributeCodec):
    '''Codec for a null-terminated string.'''

    def encode(self, item):
        return str(item) + '\0'

    def decode(self, data):
        if data[-1] != '\0':
            raise ValueError('Attribute value is not null-terminated')
        return data[:-1]


class IntegerAttributeCodec(_AttributeCodec):
    '''Codec for a 32-bit native-endian integer.'''

    def encode(self, item):
        return struct.pack('i', item)

    def decode(self, data):
        return struct.unpack('i', data)[0]


class DoubleAttributeCodec(_AttributeCodec):
    '''Codec for a native-endian double.'''

    def encode(self, item):
        return struct.pack('d', item)

    def decode(self, data):
        return struct.unpack('d', data)[0]


class RGBImageAttributeCodec(_AttributeCodec):
    '''Codec for an RGBImage structure.  Decodes to a PIL.Image.'''

    def encode(self, item):
        pixels = item.tostring('raw', 'RGBX', 0, 1)
        width, height = item.size
        header = struct.pack('IIii', 0, len(pixels) + 16, height, width)
        return header + pixels

    def decode(self, data):
        # Parse the dimensions out of the header
        height, width = struct.unpack('2i', data[8:16])
        # Read the image data
        return PIL.Image.fromstring('RGB', (width, height), data[16:],
                                'raw', 'RGBX', 0, 1)


class PatchesAttributeCodec(_AttributeCodec):
    '''Codec for a list of patches.  Decodes to (distance, patches), where
    distance is a double, patches is a tuple of (upper_left_coord,
    lower_right_coord) tuples, and a coordinate is an (x, y) tuple.'''
<commit_msg>Avoid PIL.Image.{from,to}string() deprecation warnings<commit_after>#
#  The OpenDiamond Platform for Interactive Search
#
#  Copyright (c) 2011 Carnegie Mellon University
#  All rights reserved.
#
#  This software is distributed under the terms of the Eclipse Public
#  License, Version 1.0 which can be found in the file named LICENSE.
#  ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS SOFTWARE CONSTITUTES
#  RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT
#

from cStringIO import StringIO
import PIL.Image
import struct

class _AttributeCodec(object):
    '''Base class for a Diamond attribute codec.'''

    def encode(self, item):
        raise NotImplementedError()

    def decode(self, data):
        raise NotImplementedError()


class StringAttributeCodec(_AttributeCodec):
    '''Codec for a null-terminated string.'''

    def encode(self, item):
        return str(item) + '\0'

    def decode(self, data):
        if data[-1] != '\0':
            raise ValueError('Attribute value is not null-terminated')
        return data[:-1]


class IntegerAttributeCodec(_AttributeCodec):
    '''Codec for a 32-bit native-endian integer.'''

    def encode(self, item):
        return struct.pack('i', item)

    def decode(self, data):
        return struct.unpack('i', data)[0]


class DoubleAttributeCodec(_AttributeCodec):
    '''Codec for a native-endian double.'''

    def encode(self, item):
        return struct.pack('d', item)

    def decode(self, data):
        return struct.unpack('d', data)[0]


class RGBImageAttributeCodec(_AttributeCodec):
    '''Codec for an RGBImage structure.  Decodes to a PIL.Image.'''

    def encode(self, item):
        # tobytes in Pillow >= 2.0.0, tostring in older Pillow/PIL
        tobytes = getattr(item, 'tobytes', item.tostring)
        pixels = tobytes('raw', 'RGBX', 0, 1)
        width, height = item.size
        header = struct.pack('IIii', 0, len(pixels) + 16, height, width)
        return header + pixels

    def decode(self, data):
        # Parse the dimensions out of the header
        height, width = struct.unpack('2i', data[8:16])
        # Read the image data
        # frombytes in Pillow >= 2.0.0, fromstring in older Pillow/PIL
        frombytes = getattr(PIL.Image, 'frombytes', PIL.Image.fromstring)
        return frombytes('RGB', (width, height), data[16:], 'raw', 'RGBX',
                0, 1)


class PatchesAttributeCodec(_AttributeCodec):
    '''Codec for a list of patches.  Decodes to (distance, patches), where
    distance is a double, patches is a tuple of (upper_left_coord,
    lower_right_coord) tuples, and a coordinate is an (x, y) tuple.'''

{'fraction_non_alphanumeric': 0.11200321156162184, 'fraction_numerical': 0.01204335608189482, 'mean_word_length': 3.1352697095435684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4860405', 'n_tokens_mistral': 1528, 'n_tokens_neox': 1474, 'n_words': 503}