<commit_before>import os
import re
import sys
from pathlib import Path

import pytest

import large_image
from large_image.tilesource import nearPowerOfTwo

from . import utilities
from .datastore import datastore, registry

# In general, if there is something in skipTiles, the reader should be improved
# to either indicate that the file can't be read or changed to handle reading
# with correct exceptions.
# 'skip' is used to exclude testing specific paths.  This might be necessary
# if a file is dependent on other files as these generalized tests don't ensure
# a download order.
SourceAndFiles = {
    'bioformats': {
        'read': r'\.(czi|jp2|svs|scn)$',
        # We need to modify the bioformats reader similar to tiff's
        # getTileFromEmptyDirectory
        'skipTiles': r'(JK-kidney_B|TCGA-AA-A02O|TCGA-DU-6399|sample_jp2k_33003|\.scn$)'},
    'deepzoom': {},
    'dummy': {'any': True, 'skipTiles': r''},
    'gdal': {
        'read': r'\.(jpeg|jp2|ptif|nc|scn|svs|tif.*)$',
        'noread': r'(huron\.image2_jpeg2k|sample_jp2k_33003|TCGA-DU-6399|\.(ome.tiff)$)',
        'skipTiles': r'\.*nc$'},
    'mapnik': {
        'read': r'\.(jpeg|jp2|ptif|nc|scn|svs|tif.*)$',
        'noread': r'(huron\.image2_jpeg2k|sample_jp2k_33003|TCGA-DU-6399|\.(ome.tiff)$)',
        # we should only test this with a projection
        'skipTiles': r''},
    'multi': {
        'read': r'\.(yml|yaml)$',
        'skip': r'(multi_source\.yml)$',
    },
    'ometiff': {'read': r'\.(ome\.tif.*)$'},
    'openjpeg': {'read': r'\.(jp2)$'},
    'openslide': {
        'read': r'\.(ptif|svs|tif.*)$',
        'noread': r'(oahu|DDX58_AXL|huron\.image2_jpeg2k|landcover_sample|d042-353\.crop|US_Geo\.)',
        'skipTiles': r'one_layer_missing'},
    'pil': {
        'read': r'\.(jpeg|png|tif.*)$',
        'noread': r'(G10-3|JK-kidney|d042-353|huron|one_layer_missing|US_Geo)'},
    'test': {'any': True, 'skipTiles': r''},
    'tiff': {
        'read': r'\.(ptif|scn|svs|tif.*)$',
        'noread': r'(oahu|DDX58_AXL|G10-3_pelvis_crop|'
                  r'd042-353\.crop\.small\.float|landcover_sample|US_Geo\.)',
        'skipTiles': r'(sample_image\.ptif|one_layer_missing_tiles)'},
    'vips': {
        'read': r'',
        'noread': r'\.(nc|nd2|yml|yaml|json|czi|png|svs|scn)$',
        'skipTiles': r'(sample_image\.ptif|one_layer_missing_tiles|JK-kidney_B-gal_H3_4C_1-500sec\.jp2)'},  # noqa
}
if sys.version_info >= (3, 7):
    SourceAndFiles.update({
        'nd2': {'read': r'\.(nd2)$'},
    })
else:
    # Python 3.6 has an older version of PIL that won't read some of the
    # ome.tif files.
    SourceAndFiles['pil']['noread'] = \
        r'(G10-3|JK-kidney|d042-353|huron|sample.*ome|one_layer_missing)'

<commit_msg>Further adjust canRead grid test.<commit_after>import os
import re
import sys
from pathlib import Path

import pytest

import large_image
from large_image.tilesource import nearPowerOfTwo

from . import utilities
from .datastore import datastore, registry

# In general, if there is something in skipTiles, the reader should be improved
# to either indicate that the file can't be read or changed to handle reading
# with correct exceptions.
# 'skip' is used to exclude testing specific paths.  This might be necessary
# if a file is dependent on other files as these generalized tests don't ensure
# a download order.
SourceAndFiles = {
    'bioformats': {
        'read': r'\.(czi|jp2|svs|scn)$',
        # We need to modify the bioformats reader similar to tiff's
        # getTileFromEmptyDirectory
        'skipTiles': r'(JK-kidney_B|TCGA-AA-A02O|TCGA-DU-6399|sample_jp2k_33003|\.scn$)'},
    'deepzoom': {},
    'dummy': {'any': True, 'skipTiles': r''},
    'gdal': {
        'read': r'\.(jpeg|jp2|ptif|nc|scn|svs|tif.*)$',
        'noread': r'(huron\.image2_jpeg2k|sample_jp2k_33003|TCGA-DU-6399|\.(ome.tiff)$)',
        'skipTiles': r'\.*nc$'},
    'mapnik': {
        'read': r'\.(jpeg|jp2|ptif|nc|scn|svs|tif.*)$',
        'noread': r'(huron\.image2_jpeg2k|sample_jp2k_33003|TCGA-DU-6399|\.(ome.tiff)$)',
        # we should only test this with a projection
        'skipTiles': r''},
    'multi': {
        'read': r'\.(yml|yaml)$',
        'skip': r'(multi_source\.yml)$',
    },
    'ometiff': {'read': r'\.(ome\.tif.*)$'},
    'openjpeg': {'read': r'\.(jp2)$'},
    'openslide': {
        'read': r'\.(ptif|svs|tif.*)$',
        'noread': r'(oahu|DDX58_AXL|huron\.image2_jpeg2k|landcover_sample|d042-353\.crop|US_Geo\.)',
        'skipTiles': r'one_layer_missing'},
    'pil': {
        'read': r'\.(jpeg|png|tif.*)$',
        'noread': r'(G10-3|JK-kidney|d042-353|huron|one_layer_missing|US_Geo)'},
    'test': {'any': True, 'skipTiles': r''},
    'tiff': {
        'read': r'\.(ptif|scn|svs|tif.*)$',
        'noread': r'(oahu|DDX58_AXL|G10-3_pelvis_crop|'
                  r'd042-353\.crop\.small\.float|landcover_sample|US_Geo\.)',
        'skipTiles': r'(sample_image\.ptif|one_layer_missing_tiles)'},
    'vips': {
        'read': r'',
        'noread': r'\.(nc|nd2|yml|yaml|json|czi|png|svs|scn)$',
        'skipTiles': r'(sample_image\.ptif|one_layer_missing_tiles|JK-kidney_B-gal_H3_4C_1-500sec\.jp2)'},  # noqa
}
if sys.version_info >= (3, 7):
    SourceAndFiles.update({
        'nd2': {'read': r'\.(nd2)$'},
    })
else:
    # Python 3.6 has an older version of PIL that won't read some of the
    # ome.tif files.
    SourceAndFiles['pil']['noread'] = \
        r'(G10-3|JK-kidney|d042-353|huron|sample.*ome|one_layer_missing|US_Geo)'


{'fraction_non_alphanumeric': 0.1897836756953281, 'fraction_numerical': 0.03381203417560444, 'mean_word_length': 3.2783825816485224, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7187691', 'n_tokens_mistral': 2307, 'n_tokens_neox': 2095, 'n_words': 406}