<commit_before>from collections import defaultdict
import glob
import logging
import os
import tempfile

from PIL import Image
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from eulfedora.util import RequestFailed
import magic

from readux.books import digwf
from readux.books.models import Page
from readux.fedora import ManagementRepository
from readux.utils import md5sum

logger = logging.getLogger(__name__)


class BasePageImport(BaseCommand):
    # common logic for importing covers and book pages

    def setup(self, **options):
        self.dry_run = options.get('dry_run', False)
        self.verbosity = int(options.get('verbosity', self.v_normal))

        if not hasattr(settings, 'DIGITIZATION_WORKFLOW_API'):
            raise CommandError('DIGITIZATION_WORKFLOW_API is not configured in Django settings')

        self.digwf_client = digwf.Client(settings.DIGITIZATION_WORKFLOW_API)
        self.repo = ManagementRepository()

        self.stats = defaultdict(int)

    def is_usable_volume(self, vol):
        # if object does not exist or cannot be accessed in fedora, skip it
        if not vol.exists:
            self.stats['errors'] += 1
            if self.verbosity >= self.v_normal:
                self.stdout.write('%s does not exist' % vol.pid)
            return False
        # if object is not a volume, skip it
        elif not vol.has_requisite_content_models:
            self.stats['errors'] += 1
            if self.verbosity >= self.v_normal:
                self.stdout.write('%s is not a Volume object' % vol.pid)
            return False

        return True



    def find_page_images(self, vol):
        '''Look up an item in the DigWf by pid (noid) and find display
        images.  Returns a tuple of the list of images found (empty if none
        were found) and the info returned by :class:`readux.books.digwf.Client`.

        Raises :class:`TooManyDigWFRecords` or :class:`NoDigWFRecords` if the
        digwf api returns too many or not enough records.
        '''
        images = []
        vol_info = None

        # lookup in digwf by pid (noid)
        info = self.digwf_client.get_items(pid=vol.noid)
        if info.count != 1:
            if self.verbosity >= self.v_normal:
                if info.count > 1:
                    self.stdout.write("Error: Found more than one (%d) DigWF record for %s. This shouldn't happen!" \
                                       % (info.count, vol.pid))
                else:
                    if self.verbosity >= self.v_normal:
                        self.stdout.write("Error: No information found in DigWF for %s" % vol.pid)

            # nothing to do
            return images, vol_info

        vol_info = info.items[0]
        logger.debug("image path for %s : %s" % \
           (vol.pid, vol_info.display_image_path))
        # look for JPEG2000 images first (preferred format)
        images = glob.glob(os.path.join(vol_info.display_image_path,
                                             '*.jp2'))
        # if not found in base display path, check for a JP2 subdir
        if not len(images):
            images = glob.glob(os.path.join(vol_info.display_image_path,
                                                 'JP2', '*.jp2'))
        # if jp2s were not found in either location, look for tiffs
        if not len(images):
            images = glob.glob('%s/*.tif' % vol_info.display_image_path)

        # make sure the files are sorted; images are expected to be named
        # so that they are ordered in page-sequence when sorted
        images.sort()

        if not len(images):
            self.stdout.write('Error: no files matching *.tif or *.jp2 found for %s' % \
                              vol_info.display_image_path)

        # images could be empty list if no matches were found
        return images, vol_info

    #: how many pages in to look for a cover (0-based)
    cover_range = 4

    def identify_cover(self, images):
        '''Look through the first few images; the first non-blank
        one should be the cover.

        Returns a tuple of the image filename and the index where it
        was found.
        '''
        coverfile = coverindex = None

        for index in range(0, self.cover_range):
            imgfile = images[index]
            # first non-blank page should be the cover
            if not self.is_blank_page(imgfile):
                coverfile = imgfile
                coverindex = index
                break

        return coverfile, coverindex

    def is_blank_page(self, imgfile):
        '''Check whether or not a specified image is blank.'''
        img = Image.open(imgfile, mode='r')
        colors = img.getcolors()
        # getcolors returns None if maxcolors (default=256) is exceeded
        if colors is None:
            colors = img.getcolors(1000000)  # set maxcolors ridiculously high<commit_msg>more cleanup / error cases for page import script logic<commit_after>from collections import defaultdict
import glob
import logging
import os
import tempfile

from PIL import Image
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from eulfedora.util import RequestFailed
import magic

from readux.books import digwf
from readux.books.models import Page
from readux.fedora import ManagementRepository
from readux.utils import md5sum

logger = logging.getLogger(__name__)


class BasePageImport(BaseCommand):
    # common logic for importing covers and book pages

    def setup(self, **options):
        self.dry_run = options.get('dry_run', False)
        self.verbosity = int(options.get('verbosity', self.v_normal))

        if not hasattr(settings, 'DIGITIZATION_WORKFLOW_API'):
            raise CommandError('DIGITIZATION_WORKFLOW_API is not configured in Django settings')

        self.digwf_client = digwf.Client(settings.DIGITIZATION_WORKFLOW_API)
        self.repo = ManagementRepository()

        self.stats = defaultdict(int)

    def is_usable_volume(self, vol):
        # if object does not exist or cannot be accessed in fedora, skip it
        if not vol.exists:
            self.stats['errors'] += 1
            if self.verbosity >= self.v_normal:
                self.stdout.write('%s does not exist' % vol.pid)
            return False
        # if object is not a volume, skip it
        elif not vol.has_requisite_content_models:
            self.stats['errors'] += 1
            if self.verbosity >= self.v_normal:
                self.stdout.write('%s is not a Volume object' % vol.pid)
            return False

        return True



    def find_page_images(self, vol):
        '''Look up an item in the DigWf by pid (noid) and find display
        images.  Returns a tuple of the list of images found (empty if none
        were found) and the info returned by :class:`readux.books.digwf.Client`.

        Raises :class:`TooManyDigWFRecords` or :class:`NoDigWFRecords` if the
        digwf api returns too many or not enough records.
        '''
        images = []
        vol_info = None

        # lookup in digwf by pid (noid)
        info = self.digwf_client.get_items(pid=vol.noid)
        if info.count != 1:
            if self.verbosity >= self.v_normal:
                if info.count > 1:
                    self.stdout.write("Error: Found more than one (%d) DigWF record for %s. This shouldn't happen!" \
                                       % (info.count, vol.pid))
                else:
                    if self.verbosity >= self.v_normal:
                        self.stdout.write("Error: No information found in DigWF for %s" % vol.pid)

            # nothing to do
            return images, vol_info

        vol_info = info.items[0]
        logger.debug("image path for %s : %s" % \
           (vol.pid, vol_info.display_image_path))
        # look for JPEG2000 images first (preferred format)
        images = glob.glob(os.path.join(vol_info.display_image_path,
                                             '*.jp2'))
        # if not found in base display path, check for a JP2 subdir
        if not len(images):
            images = glob.glob(os.path.join(vol_info.display_image_path,
                                                 'JP2', '*.jp2'))
        # if jp2s were not found in either location, look for tiffs
        if not len(images):
            images = glob.glob('%s/*.tif' % vol_info.display_image_path)

        # make sure the files are sorted; images are expected to be named
        # so that they are ordered in page-sequence when sorted
        images.sort()

        if not len(images):
            self.stdout.write('Error: no files matching *.tif or *.jp2 found for %s' % \
                              vol_info.display_image_path)

        # images could be empty list if no matches were found
        return images, vol_info

    #: how many pages in to look for a cover (0-based)
    cover_range = 4

    def identify_cover(self, images):
        '''Look through the first few images; the first non-blank
        one should be the cover.

        Returns a tuple of the image filename and the index where it
        was found.
        '''
        coverfile = coverindex = None

        for index in range(0, self.cover_range):
            imgfile = images[index]
            # first non-blank page should be the cover
            if not self.is_blank_page(imgfile):
                coverfile = imgfile
                coverindex = index
                break

        return coverfile, coverindex

    def is_blank_page(self, imgfile):
        '''Check whether or not a specified image is blank.'''

        # in some cases, there are empty files; consider empty == blank
        if os.path.getsize(imgfile) == 0:
            logger.debug('%s is an empty file; considering blank')
            return True

        img = Image.open(imgfile, mode='r')
        colors = img.getcolors()
        # getcolors returns None if maxcolors (default=256) is exceeded
        if colors is None:
            colors = img.getcolors(1000000)  # set maxcolors ridiculously high
            # FIXME: colors still could be none at this point
            # -- if so, can we assume not blank?
{'fraction_non_alphanumeric': 0.07991803278688525, 'fraction_numerical': 0.005757220921155348, 'mean_word_length': 2.041246290801187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '960568', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2706, 'n_words': 967}