<jupyter_start><jupyter_text>```img_size(6000, 4000)	14703(1872, 1053)	7534(640, 480)	4147(5184, 3456)	3418(3264, 2448)	1483(4288, 2848)	729(2592, 1936)	674(3888, 2592)	140(4032, 3024)	84(2317, 2317)	29(2848, 4288)	17(3456, 5184)	16(4608, 3072)	10(1761, 1761)	7(1775, 1775)	7(2329, 2329)	6(1763, 1763)	6(1769, 1769)	5(1773, 1773)	4(3872, 2592)	4``` ```	img_size(6000, 4000)	14703(1872, 1053)	7534(640, 480)	4147(5184, 3456)	3418(3264, 2448)	1483(4288, 2848)	729(2592, 1936)	674(3888, 2592)	140(4032, 3024)	84(2317, 2317)	29(2848, 4288)	17(3456, 5184)	16(4608, 3072)	10(1761, 1761)	7(1775, 1775)	7(2329, 2329)	6(1763, 1763)	6(1769, 1769)	5(1773, 1773)	4(3872, 2592)	4```<jupyter_code># img.resize((60,40))<jupyter_output><empty_output><jupyter_text>Cropping with (left, upper, right, lower)<jupyter_code>def proper_size(img):
    w,h = img.size
    if h>w:
        img = img.transpose(Image.ROTATE_90)
    check = False
    while check == False:
        if min(w,h)>1599:
            img = img.resize((w//2,h//2))
        if min(w,h)<256:
            img = img.resize((w*2,h*2))
            
        w,h = img.size
        check = True
    return img
def find_center(img,size = 256):
    w,h = img.size
    left = w//2-size//2
    upper = h//2-size//2
    right = left+size
    lower = upper+size
    return img.crop((left, upper, right, lower))
from random import choice
def find_ratio(img,size = 256,ratio = .3):
    w,h = img.size
    h2 = int(h*ratio)
    upper = (h-h2)//2
    lower = upper+h2
    
    w2 = int(w*ratio)
    wpad = (w-w2)//2
    start = choice(list(range(max(1,w2-h2))))
    left = wpad+start
    right = left+h2

    return img.crop((left, upper, right, lower)).resize((size,size))
def combine_4in1(*imgs,size = 256):
    dst = Image.new('RGB', (size*2,size*2))
    dst.paste(imgs[0], (0, 0))
    dst.paste(imgs[1], (0, size))
    dst.paste(imgs[2], (size, 0))
    dst.paste(imgs[3], (size, size))
    return dst
def different_scale_crop(img,size=512):
    img = proper_size(img)
    return combine_4in1(*map(lambda i:find_ratio(img,size=size//2,ratio = 1-2*(i/10)),range(1,5)))
img = different_scale_crop(open_img(TRAIN.ls()[0]),size = 512)
HOME = Path(".")
TRAIN_SAVE = HOME/"img/train"
TEST_SAVE = HOME/"img/test"
!mkdir -p {TRAIN_SAVE}
!mkdir -p {TEST_SAVE}

train_files = TRAIN.ls()
def process_file_train(fname):
    img = open_img(fname,parent = TRAIN)
    img = different_scale_crop(img,size = 512)
    newname = fname.split(".")[0]+".jpg"
    img.save(TRAIN_SAVE/f"{newname}")
Parallel(n_jobs=8)(delayed(process_file_test)(fname) for fname in test_files)
!ls -l {TRAIN_SAVE}|wc -l
!ls -l {TEST_SAVE}|wc -l
# different_scale_crop(open_img(TRAIN.ls()[1]),size = 512)
# different_scale_crop(open_img(TRAIN.ls()[2]),size = 512)
# find_ratio(img,ratio = .4)
!tar -czvf train_data.tar.gz {TRAIN_SAVE} > /dev/null
!rm -rf {TRAIN_SAVE}
!tar -czvf test_data.tar.gz {TEST_SAVE} > /dev/null
!rm -rf {TEST_SAVE}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18344594594594596, 'fraction_numerical': 0.16689189189189188, 'mean_word_length': 3.976470588235294, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '401201', 'license': 'Apache-2.0', 'n_tokens_mistral': 1631, 'n_tokens_neox': 1332, 'n_words': 254, 'path': 'nbs/funway-of-sampling.ipynb', 'repo_name': 'iofthetiger/siim', 'size': '2960'}