<commit_before>import argparse
import logging
import os
import sys
import time

import large_image_converter


logger = logging.getLogger('large-image-converter')


def get_parser():
    parser = argparse.ArgumentParser(description="""
Convert files for use with Large Image.
Output files are written as tiled tiff files.  For geospatial files, these
conform to the cloud-optimized geospatial tiff format (COG).  For
non-geospatial, the output image will be either 8- or 16-bits per sample per
channel.  Some compression formats are always 8-bits per sample (webp, jpeg),
even if that format could support more and the original image is higher bit
depth.
""")
    parser.add_argument(
        '--version', action='version',
        version=large_image_converter.__version__, help='Report version')
    parser.add_argument(
        '--verbose', '-v', action='count', default=0, help='Increase verbosity')
    parser.add_argument(
        '--silent', '-s', action='count', default=0, help='Decrease verbosity')
    parser.add_argument(
        'source', help='Path to source image')
    parser.add_argument(
        'dest', nargs='?', help='Output path')
    parser.add_argument(
        '--compression', '-c',
        choices=[
            '', 'jpeg', 'deflate', 'zip', 'lzw', 'zstd', 'packbits', 'jbig',
            'lzma', 'webp', 'jp2k', 'none',
        ],
        help='Internal compression.  Default will use jpeg if the source '
        'appears to be lossy or lzw if lossless.  lzw is the most compatible '
        'lossless mode.  jpeg is the most compatible lossy mode.  jbig and '
        'lzma may not be available.  jp2k will first write the file with no '
        'compression and then rewrite it with jp2k the specified psnr or '
        'compression ratio.')
    parser.add_argument(
        '--quality', '-q', type=int,
        help='JPEG or webp compression quality.  For webp, specify 0 for '
        'lossless.  Default is 90.')
    parser.add_argument(
        '--level', '-l', type=int,
        help='General compression level.  Used for deflate (zip) (1-9), zstd '
        '(1-22), and some others.')
    parser.add_argument(
        '--predictor', '-p', choices=['', 'none', 'horizontal', 'float', 'yes'],
        help='Predictor for some compressions.  Default is horizontal for '
        'non-geospatial data and yes for geospatial.')
    parser.add_argument(
        '--psnr', type=int,
        help='JP2K peak signal to noise ratio.  0 for lossless.')
    parser.add_argument(
        '--cr', type=int, help='JP2K compression ratio.  1 for lossless.')
    parser.add_argument(
        '--tile', '--tile-size', '--tilesize', '--tileSize', '-t', type=int,
        help='Tile size.  Default is 256.', dest='tileSize')
    parser.add_argument(
        '--no-subifds', action='store_false', dest='subifds', default=None,
        help='When writing multiframe files, do not use subifds.')
    parser.add_argument(
        '--subifds', action='store_true', dest='subifds', default=None,
        help='When writing multiframe files, use subifds.')
    parser.add_argument(
        '--overwrite', '-w', action='store_true',
        help='Overwrite an existing output file')
    parser.add_argument(
        '--stats', action='store_true', dest='_stats',
        help='Add conversion stats (time and size) to the ImageDescription of '
        'the output file.  This involves writing the file an extra time; the '
        'stats do not include the extra write.')
    parser.add_argument(
        '--stats-full', '--full-stats',
        action='store_const', const='full', dest='_stats',
        help='Add conversion stats, including noise metrics (PSNR, etc.) to '
        'the output file.  This takes more time and temporary disk space.')
    parser.add_argument(
        '--concurrency', '-j', type=int, dest='_concurrency',
        help='Maximum processor concurrency.  Some conversion tasks can use'
        'multiple processors.  By default, all logical processors are used.  '
        'This is a recommendation and is not strict.')
    return parser


def compute_error_metrics(original, altered, results, converterOpts=None):
    """
    Compute the amount of error introduced via conversion compared to
    conversion using a lossless method.  Note that this is not compared to the
    original, as we may not be able to read that in an efficient way.  This is
    a very time-consuming way to compute error metrics, since it first
    reprocesses the input file to a lossless format, then steps through each
    tile and computes RSME and SSIM errors per tile, producing a weighted-by-
    number-of-pixels of each of these.  The RSME is used to compute a PSNR
    value.

    :param original: the original file path.
    :param altered: the path of compressed file to compare.
    :param results: a dictionary to store results.  Modified.
    :param converterOpts: an optional dictionary of parameters used for the
        original conversion.  Only parameters that would affect the selected
        pixels are used.
    """
    import math
    from tempfile import TemporaryDirectory
    import skimage.metrics
    import numpy
    import large_image

    lastlog = 0
    with TemporaryDirectory() as tempDir:
        tempPath = os.path.join(tempDir, os.path.basename(original))
        orig = large_image_converter.convert(original, tempPath, compression='lzw')
        tsOrig = large_image.getTileSource(orig)
        numFrames = len(tsOrig.getMetadata().get('frames', [0]))
        tsAlt = large_image.getTileSource(altered)
        mse = 0
        ssim = 0<commit_msg>Sometimes the statistics weren't computed correctly.<commit_after>import argparse
import logging
import os
import sys
import time

import large_image_converter


logger = logging.getLogger('large-image-converter')


def get_parser():
    parser = argparse.ArgumentParser(description="""
Convert files for use with Large Image.
Output files are written as tiled tiff files.  For geospatial files, these
conform to the cloud-optimized geospatial tiff format (COG).  For
non-geospatial, the output image will be either 8- or 16-bits per sample per
channel.  Some compression formats are always 8-bits per sample (webp, jpeg),
even if that format could support more and the original image is higher bit
depth.
""")
    parser.add_argument(
        '--version', action='version',
        version=large_image_converter.__version__, help='Report version')
    parser.add_argument(
        '--verbose', '-v', action='count', default=0, help='Increase verbosity')
    parser.add_argument(
        '--silent', '-s', action='count', default=0, help='Decrease verbosity')
    parser.add_argument(
        'source', help='Path to source image')
    parser.add_argument(
        'dest', nargs='?', help='Output path')
    parser.add_argument(
        '--compression', '-c',
        choices=[
            '', 'jpeg', 'deflate', 'zip', 'lzw', 'zstd', 'packbits', 'jbig',
            'lzma', 'webp', 'jp2k', 'none',
        ],
        help='Internal compression.  Default will use jpeg if the source '
        'appears to be lossy or lzw if lossless.  lzw is the most compatible '
        'lossless mode.  jpeg is the most compatible lossy mode.  jbig and '
        'lzma may not be available.  jp2k will first write the file with no '
        'compression and then rewrite it with jp2k the specified psnr or '
        'compression ratio.')
    parser.add_argument(
        '--quality', '-q', type=int,
        help='JPEG or webp compression quality.  For webp, specify 0 for '
        'lossless.  Default is 90.')
    parser.add_argument(
        '--level', '-l', type=int,
        help='General compression level.  Used for deflate (zip) (1-9), zstd '
        '(1-22), and some others.')
    parser.add_argument(
        '--predictor', '-p', choices=['', 'none', 'horizontal', 'float', 'yes'],
        help='Predictor for some compressions.  Default is horizontal for '
        'non-geospatial data and yes for geospatial.')
    parser.add_argument(
        '--psnr', type=int,
        help='JP2K peak signal to noise ratio.  0 for lossless.')
    parser.add_argument(
        '--cr', type=int, help='JP2K compression ratio.  1 for lossless.')
    parser.add_argument(
        '--tile', '--tile-size', '--tilesize', '--tileSize', '-t', type=int,
        help='Tile size.  Default is 256.', dest='tileSize')
    parser.add_argument(
        '--no-subifds', action='store_false', dest='subifds', default=None,
        help='When writing multiframe files, do not use subifds.')
    parser.add_argument(
        '--subifds', action='store_true', dest='subifds', default=None,
        help='When writing multiframe files, use subifds.')
    parser.add_argument(
        '--overwrite', '-w', action='store_true',
        help='Overwrite an existing output file')
    parser.add_argument(
        '--stats', action='store_true', dest='_stats',
        help='Add conversion stats (time and size) to the ImageDescription of '
        'the output file.  This involves writing the file an extra time; the '
        'stats do not include the extra write.')
    parser.add_argument(
        '--stats-full', '--full-stats',
        action='store_const', const='full', dest='_stats',
        help='Add conversion stats, including noise metrics (PSNR, etc.) to '
        'the output file.  This takes more time and temporary disk space.')
    parser.add_argument(
        '--concurrency', '-j', type=int, dest='_concurrency',
        help='Maximum processor concurrency.  Some conversion tasks can use'
        'multiple processors.  By default, all logical processors are used.  '
        'This is a recommendation and is not strict.')
    return parser


def compute_error_metrics(original, altered, results, converterOpts=None):
    """
    Compute the amount of error introduced via conversion compared to
    conversion using a lossless method.  Note that this is not compared to the
    original, as we may not be able to read that in an efficient way.  This is
    a very time-consuming way to compute error metrics, since it first
    reprocesses the input file to a lossless format, then steps through each
    tile and computes RSME and SSIM errors per tile, producing a weighted-by-
    number-of-pixels of each of these.  The RSME is used to compute a PSNR
    value.

    :param original: the original file path.
    :param altered: the path of compressed file to compare.
    :param results: a dictionary to store results.  Modified.
    :param converterOpts: an optional dictionary of parameters used for the
        original conversion.  Only parameters that would affect the selected
        pixels are used.
    """
    import math
    from tempfile import TemporaryDirectory
    import skimage.metrics
    import numpy
    import large_image_source_tiff

    lastlog = 0
    with TemporaryDirectory() as tempDir:
        tempPath = os.path.join(tempDir, os.path.basename(original) + '.tiff')
        orig = large_image_converter.convert(original, tempPath, compression='lzw')
        tsOrig = large_image_source_tiff.TiffFileTileSource(orig)
        numFrames = len(tsOrig.getMetadata().get('frames', [0]))
        tsAlt = large_image_source_tiff.TiffFileTileSource(altered)
        mse = 0
        ssim = 0
{'fraction_non_alphanumeric': 0.10836653386454183, 'fraction_numerical': 0.004957945993802567, 'mean_word_length': 2.966292134831461, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '672615', 'n_tokens_mistral': 3289, 'n_tokens_neox': 3198, 'n_words': 1163}