<commit_before>import hashlib
import os
import shutil

import wand.image

raw_formats = ['CR2']
thumbnail_formats = ['JPG', 'jpg']


class Library(object):

    def __init__(self, library_path):
        super(Library, self).__init__()

        self.library_path = library_path
        self.hashes = []

        for root, _, files in os.walk(os.path.join(library_path, 'thumbnail')):
            for file_name in files:
                if file_name.split('.')[-1] in thumbnail_formats:
                    self.hashes.append(file_name.split('/')[-1].split('.')[0])

    def all(self):
        return self.query(lambda image: True)

    def query(self, match):
        selection = Selection()
        for hash_code in self.hashes:
            image = Image(self.library_path, hash_code)
            if match(image):
                selection.append(image)

        return selection

    def import_photos(self, path, notify=None, imported=None):
        file_list = []

        for root, _, files in os.walk(path):
            for file_name in files:
                if file_name.split('.')[-1] in raw_formats:
                    file_path = os.path.join(root, file_name)
                    file_list.append(file_path)

        num_complete = 0

        for file_path in file_list:
            # TODO: skip if already imported

            if notify:
                notify('Importing {0}...'.format(file_path))

            file_hash = self.hash_file(file_path)

            if file_hash not in self.hashes:
                # copy file
                file_name = '{file_hash}.{extension}'.format(
                    file_hash=file_hash,
                    extension=file_path.split('/')[-1].split('.')[-1],
                )
                new_file_path = os.path.join(
                    self.library_path, 'raw', file_name)
                shutil.copyfile(
                    file_path, os.path.join(self.library_path, 'raw', file_name))

                # generate jpg
                thumbnail_name = '{file_hash}.{extension}'.format(
                    file_hash=file_hash,
                    extension='jpg',
                )
                new_thumbnail_path = os.path.join(
                    self.library_path, 'thumbnail', thumbnail_name)
                with wand.image.Image(filename=new_file_path) as image:
                    with image.convert('jpeg') as thumbnail:
                        thumbnail.save(filename=new_thumbnail_path)

                self.hashes.append(file_hash)

            num_complete += 1
<commit_msg>subprocess to dcraw instead of using wand to develop raw<commit_after>import hashlib
import os
import shutil
import subprocess

import wand.image

raw_formats = ['CR2']
thumbnail_formats = ['JPG', 'jpg']


class Library(object):

    def __init__(self, library_path):
        super(Library, self).__init__()

        self.library_path = library_path
        self.hashes = []

        for root, _, files in os.walk(os.path.join(library_path, 'thumbnail')):
            for file_name in files:
                if file_name.split('.')[-1] in thumbnail_formats:
                    self.hashes.append(file_name.split('/')[-1].split('.')[0])

    def all(self):
        return self.query(lambda image: True)

    def query(self, match):
        selection = Selection()
        for hash_code in self.hashes:
            image = Image(self.library_path, hash_code)
            if match(image):
                selection.append(image)

        return selection

    def import_photos(self, path, notify=None, imported=None):
        file_list = []

        for root, _, files in os.walk(path):
            for file_name in files:
                if file_name.split('.')[-1] in raw_formats:
                    file_path = os.path.join(root, file_name)
                    file_list.append(file_path)

        num_complete = 0

        for file_path in file_list:
            # TODO: skip if already imported

            if notify:
                notify('Importing {0}...'.format(file_path))

            file_hash = self.hash_file(file_path)

            if file_hash not in self.hashes:
                # copy file
                file_name = '{file_hash}.{extension}'.format(
                    file_hash=file_hash,
                    extension=file_path.split('/')[-1].split('.')[-1],
                )
                new_file_path = os.path.join(
                    self.library_path, 'raw', file_name)
                shutil.copyfile(
                    file_path, os.path.join(self.library_path, 'raw', file_name))

                # generate jpg
                thumbnail_name = '{file_hash}.{extension}'.format(
                    file_hash=file_hash,
                    extension='jpg',
                )
                new_thumbnail_path = os.path.join(
                    self.library_path, 'thumbnail', thumbnail_name)

                # TODO: fail gracefully here (or even at startup)
                blob = subprocess.check_output(['dcraw', '-c', file_path])

                with wand.image.Image(blob=blob) as image:
                    with image.convert('jpeg') as thumbnail:
                        thumbnail.save(filename=new_thumbnail_path)

                self.hashes.append(file_hash)

            num_complete += 1

{'fraction_non_alphanumeric': 0.10708215297450425, 'fraction_numerical': 0.003777148253068933, 'mean_word_length': 1.6910569105691058, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '771840', 'n_tokens_mistral': 1546, 'n_tokens_neox': 1451, 'n_words': 313}