<jupyter_start><jupyter_text>Unfortunatly, neither of the other 2 datasets provided images in the folder form :(<jupyter_code>ships_path = pathlib.Path("/home/john/data/ships-in-satellite-imagery/shipsnet/")
image_list = os.listdir(ships_path)
s[0] == "0"
not_ship = []
is_ship = []
for s in image_list:
    if s[0] == '0':
        not_ship.append(s)
    elif s[0] == '1':
        is_ship.append(s)
    else:
        print(f"ERROR {s}")
    
        
full_data_path = ships_path/"fulldata"
full_data_path.mkdir(exist_ok=True)
for label, files in [("is_ship",is_ship),("not_ship",not_ship)]:
    folder = full_data_path/label
    folder.mkdir(exist_ok=True)
    for f in files:
        copy(ships_path/f, folder/f)
    print(f"copied {len(files)} files to {folder}")
        
create_experiment_folders(full_data_path, "/home/john/data/sat-ships/")<jupyter_output>Found 2 classes: ['is_ship', 'not_ship'] 
is_ship: train: 448 val: 112 test: 140
not_ship: train: 1344 val: 336 test: 420
Finished building experiment folders in /home/john/data/sat-ships<jupyter_text>Ok, now let's do the same for the arabic ones... We are going to extract the images from the CSV and put them into a folder. Then we will use the labels csv and the `from_csv` method of `ImageClassifierData` to extract them.<jupyter_code>from tqdm import tqdm
import shutil
PATH = pathlib.Path("/home/john/data/arabicCharacters/")
files = os.listdir(PATH)
files
trainImgs = pd.read_csv(PATH/'csvTrainImages 13440x1024.csv', header=None)
trainImgs
import matplotlib.pyplot as plt
tqdm?
imageOutputFolder = PATH/"train"
imageOutputFolder.mkdir(exist_ok=True)
for i, r in tqdm(trainImgs.iterrows(), total=13440):
    a = r.values
    a = a.reshape(32,32)
    plt.imsave(fname=str(imageOutputFolder/f"{i+1}.png"), arr=a)
testImgs = pd.read_csv(PATH/"csvTestImages 3360x1024.csv", header=None)
imageOutputFolder = PATH/"test"
offset = 13441
imageOutputFolder.mkdir(exist_ok=True)
for i, r in tqdm(testImgs.iterrows()):
    a = r.values
    a = a.reshape(32,32)
    plt.imsave(fname=str(imageOutputFolder/f"{i+offset}.png"), arr=a)
trainLabels = pd.read_csv(PATH/ 'csvTrainLabel 13440x1.csv', header=None)

output_dir = PATH/"train"
for i in range(1, 29):
    (output_dir/f"{i}").mkdir(exist_ok=True)
    
for i, r in tqdm(trainLabels.iterrows()):
    label = r[0]
    shutil.move(output_dir/f"{i+1}.png", output_dir/f"{label}"/f"{i+1}.png")
    
testLabels = pd.read_csv(PATH/ 'csvTestLabel 3360x1.csv', header=None)
output_dir = PATH/"test"
for i in range(1, 29):
    (output_dir/f"{i}").mkdir(exist_ok=True)
    
for i, r in tqdm(testLabels.iterrows()):
    label = r[0]
    shutil.move(output_dir/f"{i+offset}.png", output_dir/f"{label}"/f"{i+offset}.png")<jupyter_output>3360it [00:00, 10107.97it/s]<jupyter_text>Valid SetOk we still need to construct a validation set. Let's butcher the function from before because we have already been given a test set to work on.<jupyter_code>datapath = PATH/"train"
class_names = os.listdir(datapath)
print(f"Found {len(class_names)} classes: {class_names} ")

val = PATH/"valid"
val.mkdir(exist_ok=True)

for cl in class_names:
    elems = os.listdir(datapath/cl)
    train_set, val_set = train_test_split(elems, test_size=0.2)

    for i in tqdm(val_set):
        (val/cl).mkdir(exist_ok=True)
        shutil.move(datapath/cl/i, set_path/cl/i)<jupyter_output>100%|██████████| 96/96 [00:00<00:00, 7734.55it/s]
100%|██████████| 96/96 [00:00<00:00, 14152.02it/s]
100%|██████████| 96/96 [00:00<00:00, 14362.52it/s]
100%|██████████| 96/96 [00:00<00:00, 13714.81it/s]
100%|██████████| 96/96 [00:00<00:00, 14322.16it/s]
100%|██████████| 96/96 [00:00<00:00, 17797.61it/s]
100%|██████████| 96/96 [00:00<00:00, 16587.84it/s]
100%|██████████| 96/96 [00:00<00:00, 14089.13it/s]
100%|██████████| 96/96 [00:00<00:00, 14382.53it/s]
100%|██████████| 96/96 [00:00<00:00, 14085.19it/s]
100%|██████████| 96/96 [00:00<00:00, 14150.03it/s]
100%|██████████| 96/96 [00:00<00:00, 17341.54it/s]
100%|██████████| 96/96 [00:00<00:00, 13899.45it/s]
100%|██████████| 96/96 [00:00<00:00, 14003.87it/s]
100%|██████████| 96/96 [00:00<00:00, 14090.11it/s]
100%|██████████| 96/96 [00:00<00:00, 17271.62it/s]
100%|██████████| 96/96 [00:00<00:00, 15566.89it/s]
100%|██████████| 96/96 [00:00<00:00, 13779.11it/s]
100%|██████████| 96/96 [00:00<00:00, 17367.72it/s]
100%|██████████| 96/96 [00:00<00[...]
{'fraction_non_alphanumeric': 0.20957398076042144, 'fraction_numerical': 0.12162162162162163, 'mean_word_length': 4.893387314439946, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '480058', 'license': 'Apache-2.0', 'n_tokens_mistral': 2337, 'n_tokens_neox': 1871, 'n_words': 358, 'path': 'part_2_hints/Data Setup.ipynb', 'repo_name': 'johnny555/women-in-machine-learning', 'size': '4366'}