<commit_before>#!usr/bin/env python
import hadoopy
import StringIO
import Image
import os
import numpy as np
import picarus


def mapper(key, value):
    """
    Args:
        key, value:
           either of - the output of clustering (/partition or /samples),
                     - the output of video keyframing (/allframes)
                     - (hash,image_data) input
                     - (hash,image_metadata) input
    Env vars:
        IMAGE_TYPE: record, cluster, kv, frame
        THUMB_SIZE: longest dimension for the thumbnail images

    Yields:
        image_hash, image_data: serialized jpeg data for thumbnail
    """
    if os.environ['IMAGE_TYPE'] == 'record':
        # Image input format
        image_hash, image_metadata = key, value
        image_file = picarus.io._record_to_file(image_metadata)

    elif os.environ['IMAGE_TYPE'] == 'cluster':
        # Cluster output /partition or /sample
        cluster_index, (image_hash, image_data) = key, value
        image_file = StringIO.StringIO(image_data)

    elif os.environ['IMAGE_TYPE'] == 'kv':
        # hash, image bytes
        image_hash, image_data = key, value
        image_file = StringIO.StringIO(image_data)

    elif os.environ['IMAGE_TYPE'] == 'frame':
        (kind, image_hash), image_metadata = key, value
        if not kind == 'frame':
            return
        image_file = StringIO.StringIO(image_metadata['image_data'])

    thumb_size = int(os.environ['THUMB_SIZE'])
    image = Image.open(image_file)
    image.thumbnail((thumb_size, thumb_size))
    image = image.convert('RGB')
    s = StringIO.StringIO()
    image.save(s, 'JPEG')
    s.seek(0)

    # Output type: kv
    yield image_hash, s.buf


if __name__ == '__main__':
    hadoopy.run(mapper)<commit_msg>Fixed how make_thumbnails expects video_keyframe/allframes to look Project (tp)<commit_after>#!usr/bin/env python
import hadoopy
import StringIO
import Image
import os
import numpy as np
import picarus


def mapper(key, value):
    """
    Args:
        key, value:
           either of - the output of clustering (/partition or /samples),
                     - the output of video keyframing (/allframes)
                     - (hash,image_data) input
                     - (hash,image_metadata) input
    Env vars:
        IMAGE_TYPE: record, cluster, kv, frame
        THUMB_SIZE: longest dimension for the thumbnail images

    Yields:
        image_hash, image_data: serialized jpeg data for thumbnail
    """
    if os.environ['IMAGE_TYPE'] == 'record':
        # Image input format
        image_hash, image_metadata = key, value
        image_file = picarus.io._record_to_file(image_metadata)

    elif os.environ['IMAGE_TYPE'] == 'cluster':
        # Cluster output /partition or /sample
        cluster_index, (image_hash, image_data) = key, value
        image_file = StringIO.StringIO(image_data)

    elif os.environ['IMAGE_TYPE'] == 'kv':
        # hash, image bytes
        image_hash, image_data = key, value
        image_file = StringIO.StringIO(image_data)

    elif os.environ['IMAGE_TYPE'] == 'frame':
        image_hash, image_metadata = key, value
        image_file = StringIO.StringIO(image_metadata['image_data'])

    thumb_size = int(os.environ['THUMB_SIZE'])
    image = Image.open(image_file)
    image.thumbnail((thumb_size, thumb_size))
    image = image.convert('RGB')
    s = StringIO.StringIO()
    image.save(s, 'JPEG')
    s.seek(0)

    # Output type: kv
    yield image_hash, s.buf


if __name__ == '__main__':
    hadoopy.run(mapper)
{'fraction_non_alphanumeric': 0.10358453288173863, 'fraction_numerical': 0.0005644933672029354, 'mean_word_length': 2.634871794871795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1862960', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1067, 'n_words': 281}