<commit_before>#!/usr/bin/env python
#
# Worker module for generating thumbs and pulling exif info off images
# @author simeng
#
# -*- coding: utf-8 -*-

from redis import Redis
from PythonMagick import Image
import os
from os.path import sep
import sys
import json
from time import time
import signal
from optparse import OptionParser
from subprocess import check_call as run

class Worker:
    def __init__(self, config):
        self.config = config
        self.work_list = None
        if 'redis' in config:
            # TODO support sockets and stuffs
            if 'unix_socket_path' in config["redis"]:
                self.redis = Redis(unix_socket_path = config["redis"]["unix_socket_path"])


    def fetch_thumb_job(self):
        if self.config['fetch_mode'] == 'queue':
            return self.redis.blpop('queue:thumb')[1]
        else:
            if self.work_list == None:
                self.work_list = []

                for album in self.redis.zrange('zalbums', 0, -1):
                    for key in self.redis.zrange(album, 0, -1):
                        self.work_list.append(key)

            return self.work_list.pop()

    def get_image_info(self, key):
        image = self.redis.hgetall(key)

        return image

    def save_image_info(self, imagekey, data):
        for key, value in data.items():
            self.redis.hset(imagekey, key, value)

    def thumbnail(self, infile, outfile, size):
        #image = Image(infile)
        #image.resize(size)
        #image.write(outfile)
        resize = run(['/usr/bin/convert', '-strip',  '-thumbnail',  size, infile, outfile])
        image = Image(outfile)

        return { 'width': image.size().width(), \
                 'height': image.size().height() }

if __name__ == '__main__':
    parser = OptionParser()<commit_msg>do not upscale .gif to thumbsize, since then animations becomes huge<commit_after>#!/usr/bin/env python
#
# Worker module for generating thumbs and pulling exif info off images
# @author simeng
#
# -*- coding: utf-8 -*-

from redis import Redis
from PythonMagick import Image
import os
from os.path import sep
import sys
import json
from time import time
import signal
from optparse import OptionParser
from subprocess import check_call as run

class Worker:
    def __init__(self, config):
        self.config = config
        self.work_list = None
        if 'redis' in config:
            # TODO support sockets and stuffs
            if 'unix_socket_path' in config["redis"]:
                self.redis = Redis(unix_socket_path = config["redis"]["unix_socket_path"])


    def fetch_thumb_job(self):
        if self.config['fetch_mode'] == 'queue':
            return self.redis.blpop('queue:thumb')[1]
        else:
            if self.work_list == None:
                self.work_list = []

                for album in self.redis.zrange('zalbums', 0, -1):
                    for key in self.redis.zrange(album, 0, -1):
                        self.work_list.append(key)

            return self.work_list.pop()

    def get_image_info(self, key):
        image = self.redis.hgetall(key)

        return image

    def save_image_info(self, imagekey, data):
        for key, value in data.items():
            self.redis.hset(imagekey, key, value)

    def thumbnail(self, infile, outfile, size):
        #image = Image(infile)
        #image.resize(size)
        #image.write(outfile)
        if infile.endswith('.gif'):
            resize = run(['/usr/bin/convert', '-strip',  '-thumbnail',  size+'>"', infile, outfile])
        else:
            resize = run(['/usr/bin/convert', '-strip',  '-thumbnail',  size, infile, outfile])
        image = Image(outfile)

        return { 'width': image.size().width(), \
                 'height': image.size().height() }

if __name__ == '__main__':
    parser = OptionParser()
{'fraction_non_alphanumeric': 0.11856474258970359, 'fraction_numerical': 0.0031201248049922, 'mean_word_length': 2.310671256454389, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2676893', 'n_tokens_mistral': 1189, 'n_tokens_neox': 1150, 'n_words': 319}