<commit_before>'''
Created on Aug 3, 2012

@package: internationalization
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: <NAME>

Thumbnail manager class definition.
'''
# --------------------------------------------------------------------

from ally.container.ioc import injected
from superdesk.media_archive.api.meta_data import MetaData
from superdesk.media_archive.core.spec import IThumbnailManager, IThumbnailCreator
from cdm.spec import ICDM, PathNotFound
from os.path import join, isfile, exists, isdir, normpath, dirname
from ally.support.util_io import timestampURI
from shutil import copyfile
from ally.exception import DevelError
from superdesk.media_archive.meta.meta_data import ThumbnailFormat
from ally.container import wire
from ally.support.sqlalchemy.session import SessionSupport
from os import access, W_OK, makedirs
from collections import OrderedDict
from subprocess import Popen

# --------------------------------------------------------------------

ORIGINAL_SIZE = 'original'
# Provides the size name for the original sized images from which the thumbnails are created

# --------------------------------------------------------------------

@injected
class ThumbnailManager(SessionSupport, IThumbnailManager):
    '''
    Implementation for @see: IThumbnailManager
    '''
    thumbnail_dir_path = join('workspace', 'media_archive', 'process_thumbnail'); wire.config('thumbnail_dir_path', doc='''
    The folder path where the thumbnail images are placed for resizing''')
    # path to the thumbnail repository
    original_width = 512; wire.config('original_width', doc='''
    The width of the original thumbnail from which other sizes are generated''')
    original_height = 512; wire.config('original_height', doc='''
    The width of the original thumbnail from which other sizes are generated''')

    thumbnailSizes = {}
    # dictionary containing thumbnail sizes
    # This is basically just a simple dictionary{string, tuple(integer, integer)} that has
    # as key a path safe name and as a value a tuple with the width/height of the thumbnail.
    # example: {'small': [100, 100]}
    thumbnailCreator = IThumbnailCreator
    # object that does the actual thumbnail generation from the original image
    cdm = ICDM
    # the content delivery manager where to publish thumbnails

    def __init__(self):
        assert isinstance(self.thumbnail_dir_path, str), 'Invalid processing directory %s' % self.thumbnail_dir_path
        assert isinstance(self.thumbnailSizes, dict), 'Invalid thumbnail sizes %s' % self.thumbnailSizes
        assert isinstance(self.thumbnailCreator, IThumbnailCreator), 'Invalid thumbnail creator %s' % self.thumbnailCreator
        assert isinstance(self.cdm, ICDM), 'Invalid thumbnail CDM %s' % self.cdm

        if not exists(self.thumbnail_dir_path): makedirs(self.thumbnail_dir_path)
        if not isdir(self.thumbnail_dir_path) or not access(self.thumbnail_dir_path, W_OK):
            raise IOError('Unable to access the processing directory %s' % self.thumbnail_dir_path)

        SessionSupport.__init__(self)
        # We order the thumbnail sizes in descending order
        self.thumbnailSizes[ORIGINAL_SIZE] = [self.original_width, self.original_height]
        thumbnailSizes = [(key, sizes) for key, sizes in self.thumbnailSizes.items()]
        thumbnailSizes.sort(key=lambda pack: pack[1][0] * pack[1][1])
        self.thumbnailSizes = OrderedDict(thumbnailSizes)
        self._cache_thumbnail = {}

    def populate(self, metaData, scheme, thumbSize=None):
        assert not metaData or isinstance(metaData, MetaData), 'Invalid metaData %s' % metaData
        thumbSize = thumbSize if thumbSize else ORIGINAL_SIZE
        assert isinstance(thumbSize, str) and thumbSize in self.thumbnailSizes, \
        'Invalid size value %s' % thumbSize

        if not metaData.thumbnailFormatId:
            return metaData
        keys = {'id': metaData.Id, 'name': metaData.Name}
        if thumbSize:
            keys['size'] = thumbSize
        elif self.thumbnailSizes:
            keys['size'] = next(iter(self.thumbnailSizes))
        else:
            keys['size'] = ORIGINAL_SIZE
        thumbPath = self._getFormat(metaData.thumbnailFormatId) % keys
        try: self.cdm.getTimestamp(thumbPath)
        except PathNotFound:
            if keys['size'] == ORIGINAL_SIZE: raise DevelError('Unable to find a thumbnail for %s' % scheme)
            originalImagePath = self._reference(metaData.thumbnailFormatId, metaData.Id, metaData.Name)
            self.processThumbnail(metaData.thumbnailFormatId, originalImagePath, metaData, keys['size'])
        metaData.Thumbnail = self.cdm.getURI(thumbPath, scheme)
        return metaData

    def processThumbnail(self, thumbnailFormatId, imagePath, metaData=None, size=None):
        '''
        @see IThumbnailManager.processThumbnail
        '''
        assert isinstance(thumbnailFormatId, int), \
        'Invalid thumbnail format identifier %s' % thumbnailFormatId
        assert isinstance(imagePath, str) and isfile(imagePath), 'Invalid file path %s' % imagePath
        assert not metaData or isinstance(metaData, MetaData), 'Invalid metaData %s' % metaData
        size = size if size else ORIGINAL_SIZE
        assert isinstance(size, str) and size in self.thumbnailSizes, \
        'Invalid size value %s' % size

        keys = {} if not metaData else {'id': metaData.Id, 'name': metaData.Name}
        if size:
            keys['size'] = size
        elif self.thumbnailSizes:
            keys['size'] = next(iter(self.thumbnailSizes))
        else:
            keys['size'] = ORIGINAL_SIZE
        thumbPath = self._getFormat(thumbnailFormatId) % keys
        thumbTimestamp = self.timestampThumbnail(thumbnailFormatId, metaData, keys['size'])
        if not thumbTimestamp or thumbTimestamp < timestampURI(imagePath):
            thumbRepoPath = normpath(join(self.thumbnail_dir_path, thumbPath))
            if not isdir(dirname(thumbRepoPath)): makedirs(dirname(thumbRepoPath))
            copyfile(imagePath, thumbRepoPath)
            self.thumbnailCreator.createThumbnail(thumbRepoPath,
                                                  self.thumbnailSizes[size][0],
                                                  self.thumbnailSizes[size][0])
            self.cdm.publishFromFile(thumbPath, thumbRepoPath)

    def timestampThumbnail(self, thumbnailFormatId, metaData=None, size=None):
        '''
        @see: IThumbnailManager.timestampThumbnail
        '''
        assert isinstance(thumbnailFormatId, int), \
        'Invalid thumbnail format identifier %s' % thumbnailFormatId
        assert not metaData or isinstance(metaData, MetaData), 'Invalid metaData %s' % metaData
        size = size if size else ORIGINAL_SIZE
        assert isinstance(size, str) and size in self.thumbnailSizes, \
        'Invalid size value %s' % size

        if not metaData: metaDataId, metaDataName = None, None
        else: metaDataId, metaDataName = metaData.Id, metaData.Name
        try:
            return self.cdm.getTimestamp(self._reference(thumbnailFormatId, metaDataId, metaDataName, size))
        except PathNotFound:
            return None<commit_msg>NR-234 : Media Archive - thumbnail generator<commit_after>'''
Created on Aug 3, 2012

@package: internationalization
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: <NAME>

Thumbnail manager class definition.
'''
# --------------------------------------------------------------------

from ally.container.ioc import injected
from superdesk.media_archive.api.meta_data import MetaData
from superdesk.media_archive.core.spec import IThumbnailManager, IThumbnailCreator
from cdm.spec import ICDM, PathNotFound
from os.path import join, isfile, exists, isdir, dirname
from ally.support.util_io import timestampURI
from shutil import copyfile
from ally.exception import DevelError
from superdesk.media_archive.meta.meta_data import ThumbnailFormat
from ally.container import wire
from ally.support.sqlalchemy.session import SessionSupport
from os import access, W_OK, makedirs
from collections import OrderedDict
from subprocess import Popen
from ally.zip.util_zip import normOSPath

# --------------------------------------------------------------------

ORIGINAL_SIZE = 'original'
# Provides the size name for the original sized images from which the thumbnails are created

# --------------------------------------------------------------------

@injected
class ThumbnailManager(SessionSupport, IThumbnailManager):
    '''
    Implementation for @see: IThumbnailManager
    '''
    thumbnail_dir_path = join('workspace', 'media_archive', 'process_thumbnail'); wire.config('thumbnail_dir_path', doc='''
    The folder path where the thumbnail images are placed for resizing''')
    # path to the thumbnail repository
    original_width = 512; wire.config('original_width', doc='''
    The width of the original thumbnail from which other sizes are generated''')
    original_height = 512; wire.config('original_height', doc='''
    The width of the original thumbnail from which other sizes are generated''')

    thumbnailSizes = {}
    # dictionary containing thumbnail sizes
    # This is basically just a simple dictionary{string, tuple(integer, integer)} that has
    # as key a path safe name and as a value a tuple with the width/height of the thumbnail.
    # example: {'small': [100, 100]}
    thumbnailCreator = IThumbnailCreator
    # object that does the actual thumbnail generation from the original image
    cdm = ICDM
    # the content delivery manager where to publish thumbnails

    def __init__(self):
        assert isinstance(self.thumbnail_dir_path, str), 'Invalid processing directory %s' % self.thumbnail_dir_path
        assert isinstance(self.thumbnailSizes, dict), 'Invalid thumbnail sizes %s' % self.thumbnailSizes
        assert isinstance(self.thumbnailCreator, IThumbnailCreator), 'Invalid thumbnail creator %s' % self.thumbnailCreator
        assert isinstance(self.cdm, ICDM), 'Invalid thumbnail CDM %s' % self.cdm

        if not exists(self.thumbnail_dir_path): makedirs(self.thumbnail_dir_path)
        if not isdir(self.thumbnail_dir_path) or not access(self.thumbnail_dir_path, W_OK):
            raise IOError('Unable to access the processing directory %s' % self.thumbnail_dir_path)

        SessionSupport.__init__(self)
        # We order the thumbnail sizes in descending order
        self.thumbnailSizes[ORIGINAL_SIZE] = [self.original_width, self.original_height]
        thumbnailSizes = [(key, sizes) for key, sizes in self.thumbnailSizes.items()]
        thumbnailSizes.sort(key=lambda pack: pack[1][0] * pack[1][1])
        self.thumbnailSizes = OrderedDict(thumbnailSizes)
        self._cache_thumbnail = {}

    def populate(self, metaData, scheme, thumbSize=None):
        assert isinstance(metaData, MetaData), 'Invalid metaData %s' % metaData
        thumbSize = thumbSize if thumbSize else ORIGINAL_SIZE
        assert isinstance(thumbSize, str) and thumbSize in self.thumbnailSizes, \
        'Invalid size value %s' % thumbSize

        if not metaData.thumbnailFormatId:
            return metaData
        thumbPath = self._reference(metaData.thumbnailFormatId, metaData.Id, metaData.Name, thumbSize)
        try: self.cdm.getTimestamp(thumbPath)
        except PathNotFound:
            if thumbSize == ORIGINAL_SIZE: raise DevelError('Unable to find a thumbnail for %s' % scheme)
            originalImagePath = self._reference(metaData.thumbnailFormatId, metaData.Id, metaData.Name)
            self.processThumbnail(metaData.thumbnailFormatId, originalImagePath, metaData, thumbSize)
        metaData.Thumbnail = self.cdm.getURI(thumbPath, scheme)
        return metaData

    def processThumbnail(self, thumbnailFormatId, imagePath, metaData=None, size=None):
        '''
        @see IThumbnailManager.processThumbnail
        '''
        assert isinstance(thumbnailFormatId, int), \
        'Invalid thumbnail format identifier %s' % thumbnailFormatId
        assert isinstance(imagePath, str) and isfile(imagePath), 'Invalid file path %s' % imagePath
        assert not metaData or isinstance(metaData, MetaData), 'Invalid metaData %s' % metaData
        size = size if size else ORIGINAL_SIZE
        assert isinstance(size, str) and size in self.thumbnailSizes, \
        'Invalid size value %s' % size

        (metaDataId, metaDataName) = (None, None) if not metaData else (metaData.Id, metaData.Name)
        thumbPath = self._reference(thumbnailFormatId, metaDataId, metaDataName, size)
        thumbTimestamp = self.timestampThumbnail(thumbnailFormatId, metaData, size)
        if not thumbTimestamp or thumbTimestamp < timestampURI(imagePath):
            thumbRepoPath = normOSPath(join(self.thumbnail_dir_path, thumbPath))
            if not isdir(dirname(thumbRepoPath)): makedirs(dirname(thumbRepoPath))
            copyfile(imagePath, thumbRepoPath)
            self.thumbnailCreator.createThumbnail(thumbRepoPath,
                                                  self.thumbnailSizes[size][0],
                                                  self.thumbnailSizes[size][0])
            self.cdm.publishFromFile(thumbPath, thumbRepoPath)

    def timestampThumbnail(self, thumbnailFormatId, metaData=None, size=None):
        '''
        @see: IThumbnailManager.timestampThumbnail
        '''
        assert isinstance(thumbnailFormatId, int), \
        'Invalid thumbnail format identifier %s' % thumbnailFormatId
        assert not metaData or isinstance(metaData, MetaData), 'Invalid metaData %s' % metaData
        size = size if size else ORIGINAL_SIZE
        assert isinstance(size, str) and size in self.thumbnailSizes, \
        'Invalid size value %s' % size

        if not metaData: metaDataId, metaDataName = None, None
        else: metaDataId, metaDataName = metaData.Id, metaData.Name
        try:
            return self.cdm.getTimestamp(self._reference(thumbnailFormatId, metaDataId, metaDataName, size))
        except PathNotFound:
            return None
{'fraction_non_alphanumeric': 0.11089920180829271, 'fraction_numerical': 0.0043088224906406725, 'mean_word_length': 3.0370687197034503, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '263429', 'n_tokens_mistral': 3868, 'n_tokens_neox': 3825, 'n_words': 1101}