<commit_before>"""
Multi-page TIFF parser

"""

from donomo.archive       import models, operations
from donomo.archive.utils import image
from django.conf          import settings

import glob
import os

DEFAULT_INPUTS  = (
    models.AssetClass.UPLOAD,
    )

DEFAULT_OUTPUTS = (
    models.AssetClass.DOCUMENT,
    models.AssetClass.PAGE_ORIGINAL,
    models.AssetClass.PAGE_IMAGE,
    models.AssetClass.PAGE_THUMBNAIL,
    )

DEFAULT_ACCEPTED_MIME_TYPES = (
    models.MimeType.TIFF,
    )

##############################################################################

def handle_work_item(processor, item):

    """ Pick up a (possibly) multipage TIFF upload and turn it into a
        document having (possibly) multiple individual pages.

    """

    new_work    = []
    asset       = item['Asset-Instance']
    local_path  = item['Local-Path']
    work_dir    = os.path.dirname(local_path)
    page_prefix = os.path.join(work_dir, 'page-')

    os.system( 'tiffsplit %r %r' % (local_path, page_prefix) )

    document = operations.create_document(
        owner = asset.owner,
        title = 'Uploaded on %s (%s)' % (
            asset.date_created,
            asset.producer.process ))

    position = 1
    all_page_files = glob.glob('%s*.tif*' % page_prefix)
    all_page_files.sort()
    for page_tiff_path in all_page_files:
        new_work.extend(
            handle_page(
                processor,
                asset,
                document,
                page_tiff_path,
                position ))
        position += 1

    if document is not None:
        new_work.append(
            document.assets.get(
                asset_class__name = AssetClass.DOCUMENT,
                mime_type__name   = MimeType.BINARY ))

    return new_work

##############################################################################

def handle_page(
    processor,
    parent_asset,
    document,
    tiff_original_path,
    position ):
<commit_msg>fixing syntax error<commit_after>"""
Multi-page TIFF parser

"""

from donomo.archive       import models, operations
from donomo.archive.utils import image
from django.conf          import settings

import glob
import os

DEFAULT_INPUTS  = (
    models.AssetClass.UPLOAD,
    )

DEFAULT_OUTPUTS = (
    models.AssetClass.DOCUMENT,
    models.AssetClass.PAGE_ORIGINAL,
    models.AssetClass.PAGE_IMAGE,
    models.AssetClass.PAGE_THUMBNAIL,
    )

DEFAULT_ACCEPTED_MIME_TYPES = (
    models.MimeType.TIFF,
    )

##############################################################################

def handle_work_item(processor, item):

    """ Pick up a (possibly) multipage TIFF upload and turn it into a
        document having (possibly) multiple individual pages.

    """

    new_work    = []
    asset       = item['Asset-Instance']
    local_path  = item['Local-Path']
    work_dir    = os.path.dirname(local_path)
    page_prefix = os.path.join(work_dir, 'page-')

    os.system( 'tiffsplit %r %r' % (local_path, page_prefix) )

    document = operations.create_document(
        owner = asset.owner,
        title = 'Uploaded on %s (%s)' % (
            asset.date_created,
            asset.producer.process ))

    position = 1
    all_page_files = glob.glob('%s*.tif*' % page_prefix)
    all_page_files.sort()
    for page_tiff_path in all_page_files:
        new_work.extend(
            handle_page(
                processor,
                asset,
                document,
                page_tiff_path,
                position ))
        position += 1

    if document is not None:
        new_work.append(
            document.assets.get(
                asset_class__name = models.AssetClass.DOCUMENT,
                mime_type__name   = models.MimeType.BINARY ))

    return new_work

##############################################################################

def handle_page(
    processor,
    parent_asset,
    document,
    tiff_original_path,
    position ):

{'fraction_non_alphanumeric': 0.16262626262626262, 'fraction_numerical': 0.00101010101010101, 'mean_word_length': 2.4776119402985075, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '527880', 'n_tokens_mistral': 1199, 'n_tokens_neox': 1141, 'n_words': 236}