<commit_before>"""
Download zipfiles with the test suite of freeimage.
We also use it to test reading from a zipfile.
"""
import os
import sys

from urllib.request import urlopen
import zipfile
import shutil

import imageio

ulr = "http://sourceforge.net/projects/freeimage/files/Test%20Suite%20%28graphics%29/2.5.0/"

names = ['png', 'jpeg', 'bmp', 'ico',  'tiff', 'targa', 'koa', 'mng', 'iff', 'psd', 'ppm', 'pcx'] 

NOT_WRITABLE = ['.pgm', '.koa', '.pcx', '.mng', '.iff', '.psd', '.lbm']


if __name__ == '__main__':
    
    THISDIR = os.path.dirname(os.path.abspath(__file__))
    TESTDIR = os.path.join(THISDIR, 'temp')
    ZIPDIR = os.path.join(THISDIR, 'zipped')
    
    if not os.path.isdir(TESTDIR):
        os.mkdir(TESTDIR)
    if not os.path.isdir(ZIPDIR):
        os.mkdir(ZIPDIR)
    
    for name in names:
        fname = os.path.join(ZIPDIR, name+'.zip')
        # Make sure that the file is there
        if not os.path.isfile(fname):
            print('Downloading %s.zip' % name)
            f1 = urlopen(ulr+name+'.zip')
            f2 = open(fname, 'wb')
            shutil.copyfileobj(f1, f2)
            f1.close()
            f2.close()
        
        # Check contents
        zf = zipfile.ZipFile(fname, 'r')
        subnames = zf.namelist()
        zf.extractall(TESTDIR)
        zf.close()
        
        # Read and write each one
        for subname in subnames:
            fname_zip = fname+'/%s' % subname
            subname_, ext = os.path.splitext(subname)
            fname_dst1 = os.path.join(TESTDIR, subname+'_1'+ext)
            fname_dst2 = os.path.join(TESTDIR, subname+'_2'+ext)
            if os.path.splitext(subname)[1].lower() in NOT_WRITABLE:
                fname_dst1 += '.png'
                fname_dst2 += '.png'
            print('Reading+saving %s' % subname)
            try:
                # Read from zip, save to file
                im = imageio.imread(fname_zip)
                imageio.imsave(fname_dst1, im)
                # Read from file, save to file
                im = imageio.imread(fname_dst1)
                imageio.imsave(fname_dst2, im)
            except Exception as err:
                print('woops! ' + fname_zip)
                print('  '+str(err))<commit_msg>Pass all freeimage tests on Linux too.<commit_after>"""
Download zipfiles with the test suite of freeimage.
We also use it to test reading from a zipfile.
"""
import os
import sys

from urllib.request import urlopen
import zipfile
import shutil

import imageio


# Url to download images from
ulr = "http://sourceforge.net/projects/freeimage/files/Test%20Suite%20%28graphics%29/2.5.0/"

# Names of zipfiles to download
names = ['png', 'jpeg', 'bmp', 'ico',  'tiff', 'targa', 'koa', 'mng', 'iff', 'psd', 'ppm', 'pcx'] 

# Define what formats are not writable, we will convert these to png
NOT_WRITABLE = ['.pgm', '.koa', '.pcx', '.mng', '.iff', '.psd', '.lbm']

# Define files that fail (i.e. crash)
FAILS = []
if sys.platform.startswith('linux'):
    FAILS.extend( ['quad-jpeg.tif', 'test1g.tif'] )


if __name__ == '__main__':
    
    THISDIR = os.path.dirname(os.path.abspath(__file__))
    TESTDIR = os.path.join(THISDIR, 'temp')
    ZIPDIR = os.path.join(THISDIR, 'zipped')
    
    if not os.path.isdir(TESTDIR):
        os.mkdir(TESTDIR)
    if not os.path.isdir(ZIPDIR):
        os.mkdir(ZIPDIR)
    
    for name in names:
        fname = os.path.join(ZIPDIR, name+'.zip')
        # Make sure that the file is there
        if not os.path.isfile(fname):
            print('Downloading %s.zip' % name)
            f1 = urlopen(ulr+name+'.zip')
            f2 = open(fname, 'wb')
            shutil.copyfileobj(f1, f2)
            f1.close()
            f2.close()
        
        # Check contents
        zf = zipfile.ZipFile(fname, 'r')
        subnames = zf.namelist()
        zf.extractall(TESTDIR)
        zf.close()
        
        # Read and write each one
        for subname in subnames:
            if subname in FAILS:
                continue
            fname_zip = fname+'/%s' % subname
            subname_, ext = os.path.splitext(subname)
            fname_dst1 = os.path.join(TESTDIR, subname+'_1'+ext)
            fname_dst2 = os.path.join(TESTDIR, subname+'_2'+ext)
            if os.path.splitext(subname)[1].lower() in NOT_WRITABLE:
                fname_dst1 += '.png'
                fname_dst2 += '.png'
            print('Reading+saving %s' % subname)
            try:
                # Read from zip, save to file
                im = imageio.imread(fname_zip)
                imageio.imsave(fname_dst1, im)
                # Read from file, save to file
                im = imageio.imread(fname_dst1)
                imageio.imsave(fname_dst2, im)
            except Exception as err:
                print('woops! ' + fname_zip)
                print('  '+str(err))
{'fraction_non_alphanumeric': 0.1339653751030503, 'fraction_numerical': 0.01133553173948887, 'mean_word_length': 2.0030940594059405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4704438', 'n_tokens_mistral': 1709, 'n_tokens_neox': 1646, 'n_words': 392}