<jupyter_start><jupyter_text>Main datasetThe final dataset is 50K images multiple labled plant dataset.<jupyter_code># import image data and combine labels
def load_data(image_size, image_dir):
    X = []
    y=[]
    labels =[]
    for root, folder, files in os.walk(image_dir):
        #print( folder)
        for f in files:
            #print(f)
            if f.lower().endswith('.jpg') or f.lower().endswith('.png') or f.lower().endswith('.jpeg'):
                #print(root, folder, f)
                img = load_img(f'{root}/{f}', target_size=(image_size,image_size,3))
                img_array = img_to_array(img, dtype='uint8')
                X.append(img_array)
                
                # get y
                specie, classname = root[20:].split('___')
                specie = specie.replace('_', ' ')
                classname = classname.replace('_', ' ')
                if classname[:len(specie)].lower() != specie.lower():
                    classname=specie +' ' + classname
                #print(species.index(specie), classes.index(classname))
                y.append([species.index(specie), classes.index(classname)])
                labels.append([specie, classname])
                
    return np.array(X, dtype=np.uint8), np.array(y), np.array(labels)
data_x, data_y, labels = load_data(224, 'data/multipleplants')
plt.figure(figsize=(12,12))
plt.hist(labels[:,0], bins = len(species)) 
plt.xticks(rotation=45)
plt.show()
plt.figure(figsize=(16,16))
plt.hist(labels[:,1], bins = len(classes)) 
plt.xticks(rotation='vertical')
plt.show()
np.histogram(data_y[:,0], bins = np.arange(len(species)))
np.histogram(data_y[:,1], bins = np.arange(len(classes)))
def load_image(filename,show=True):
    #print(filename)
    img = load_img(filename, target_size=(224,224,3))
    img_array = img_to_array(img, dtype='uint8')
    if show:
        plt.imshow(img_array)
    else:
        return img_array
def show_images_matrix(image_list, nrows = 10, ncols =10, random = False):
    fig, axes = plt.subplots(figsize=(50 ,50), nrows=nrows, ncols=10)
    if random:
        img_filename = random.choice(image_list)
    else:
        img_id = 0
        img_filename = image_list[img_id]
        
    for ax in axes.ravel():
        #image = load_image(f'data/fgvc7/images/{img_filename}.jpg', False)
        img = load_img(f'data/fgvc7/images/{img_filename}.jpg', target_size=(50,50,3))
        img_array = img_to_array(img, dtype='uint8')
        if random:
            img_filename = random.choice(image_list)
        else:
            img_id += 1
            img_filename = image_list[img_id]

        ax.imshow(img_array)
        ax.axis('off')
load_image(r'F:\notebooks\capstone\data\multipleplants\Apple___Cedar_apple_rust\0a41c25a-f9a6-4c34-8e5c-7f89a6ac4c40___FREC_C.Rust 9807.JPG')
healthy_list = train_df[train_df["healthy"] ==1]['image_id'].to_numpy()
print(len(healthy_list))
show_images_matrix(healthy_list)<jupyter_output><empty_output><jupyter_text>Augmentation<jupyter_code># Use data augmentation
import random
import imgaug as ia
import imgaug.augmenters as iaa

def do_augmentation(X_train):
    folder_path ='debug'
    seq = iaa.Sequential([
        iaa.SomeOf((1,3),[
            iaa.Identity(),
            iaa.Affine(translate_px={"x": (-30, 30), "y": (-30, 30)}),
            iaa.Rotate((-30,30)),
            iaa.Affine(scale=(0.8, 1.1)),
            iaa.Affine(shear=(-7, 7)),
            iaa.Dropout(p=(0, 0.1)),
            iaa.CropAndPad(percent=(-0.25, 0.25)),
            iaa.PiecewiseAffine(scale=(0.01, 0.05)),
            iaa.GaussianBlur(sigma=(0.0, 1.5)),
            iaa.GammaContrast((0.5, 2.0)),
            iaa.Multiply((0.5, 1.5))
           ])
            
    ])
    #seq_det = seq.to_deterministic()
    X_train_aug = np.zeros((X_train.shape),dtype='uint8')
    
    for i in range(len(X_train)):
        image_aug= seq(image=X_train[i])
        X_train_aug[i] = image_aug
    return X_train_aug

def generator(features, labels, batch_size):
    # create empty arrays to contain batch of features and labels
    batch_features = np.zeros((batch_size, features.shape[1], features.shape[2],3))
    
    # convert labels to keypoints
    while True:
        # Fill arrays of batch size with augmented data taken randomly from full passed arrays
        indexes = random.sample(range(len(features)), batch_size)
        # Perform the exactly the same augmentation for X and y
        random_augmented_images = do_augmentation(features[indexes])

        yield random_augmented_images, labels[indexes]
        
def show_generator_images(X,y):
    fig, axes = plt.subplots(figsize=(20, 20), nrows=6, ncols=5)
    reader= generator(X, y, 30)

    for ax in axes.ravel():
        v = next(reader)
        image = v[0][0]

        ax.imshow(image)
        ax.axis('off')
show_generator_images(X_raw, y_raw)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1417483660130719, 'fraction_numerical': 0.02798202614379085, 'mean_word_length': 2.7961240310077518, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '103078', 'license': 'MIT', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1669, 'n_words': 307, 'path': '1. EDA-All-Plants.ipynb', 'repo_name': 'louis-li/PlantDiseaseDetection', 'size': '4896'}