<jupyter_start><jupyter_text>El planPara cada observación hay que asociar las etiquetas de especies. Esto requiere construir un indice de coordenadas para asociar al patch respectivo todas las etiquetas necesarias.1. Crear datablock para leer las imagenes rgb:    1. Como leer las etiquetas    1. Cómo asociar los archivos de imagen1.<jupyter_code>#load_patch??<jupyter_output><empty_output><jupyter_text>Obtener la ruta de archivo<jupyter_code>def get_x(r):
    """Loads the patch data associated to an observation id

    Parameters
    ----------
    observation_id : integer
        Identifier of the observation.
    patches_path : string / pathlib.Path
        Path to the folder containing all the patches.
    landcover_mapping : 1d array-like
        Facultative mapping of landcover codes, useful to align France and US codes.
    return_arrays : boolean
        If True, returns all the patches as Numpy arrays (no PIL.Image returned).

    Returns
    -------
    patches : tuple of size 4 containing 2d array-like objects
        Returns a tuple containing all the patches in the following order: RGB, Near-IR, altitude and landcover.
    """
    observation_id = str(r['observation_id'])

    region_id = observation_id[0]
    if region_id == "1":
        region = "fr"
    elif region_id == "2":
        region = "us"
    else:
        raise ValueError("Incorrect 'observation_id' {}, can not extract region id from it".format(observation_id))

    subfolder1 = observation_id[-2:]
    subfolder2 = observation_id[-4:-2]

    filename = Path(PATCHES_PATH) / region / subfolder1 / subfolder2 / observation_id

    rgb_filename = filename.with_name(filename.stem + "_rgb.jpg")
    
    return rgb_filename

def get_y(r):
    species_id = r['species_id']
    
    return species_id

def splitter(df):
    train = df.index[df['subset'] == 'train'].tolist()
    valid = df.index[df['subset'] != 'train'].tolist()
    return train,valid

def get_observations(data_path):
    df_fr = pd.read_csv(data_path / "observations" / "observations_fr_train.csv",
                        sep=";", index_col="observation_id")
    df_us = pd.read_csv(data_path / "observations" / "observations_us_train.csv",
                        sep=";", index_col="observation_id")
    
    df = pd.concat((df_fr, df_us))
    
    return df

def generate_submission_file(filename, corrected_observation_ids, s_pred):
    s_pred = [
        " ".join(map(str, pred_set))
        for pred_set in s_pred
    ]
    
    df = pd.DataFrame({
        "Id": corrected_observation_ids,
        "Predicted": s_pred
    })
    df.to_csv(filename, index=False)
    
def get_test_observations(data_path):
    df_fr_test = pd.read_csv(DATA_PATH / "observations" / "observations_fr_test.csv", sep=";",
                             low_memory=False, dtype={'observation_id': str})
    df_us_test = pd.read_csv(DATA_PATH / "observations" / "observations_us_test.csv", sep=";",
                             low_memory=False, dtype={'observation_id': str})
    
    df_test = pd.concat((df_fr_test, df_us_test))
    return df_test
def training_setup(load_model=True, load_dls= True, data_limit = None, model_file_name='geolife-21-cnn'):            
    save_cbs = SaveModelCallback(every_epoch=True, with_opt=True, fname=model_file_name)
    
    if load_model:
        load_dls = True
    
    if load_dls: 
        dls = load_pickle(f'{model_file_name}_dataloaders.pkl')

    if load_model:
        learn = cnn_learner(dls, resnet34, cbs=save_cbs)
        
        model_file = get_last_model_file(model_file_name)
        learn = learn.load(model_file)
    else:
        
        if not load_dls:
            df = get_observations(DATA_PATH)
            df = df.reset_index()

            if data_limit != None:
                df = df.sample(n=data_limit, random_state = 1)
                df = df.reset_index(drop=True)
            
            not_train_labels = set(df[df.subset == 'val'].species_id).difference(df[df.subset == 'train'].species_id)
            cond = df.species_id.isin(not_train_labels)
            remove_idx = np.where(cond)[0]

            df = df.drop(index=remove_idx)
            df = df.reset_index(drop=True)

            dblock = DataBlock((ImageBlock, CategoryBlock),
                       get_x = get_x,
                       get_y = get_y,
                       splitter=splitter)

            dls = dblock.dataloaders(df, bs=256)
            save_pickle(f'{model_file_name}_dataloaders.pkl', dls)
            save_pickle(f'{model_file_name}_observations_train_df.pkl', df)

        learn = cnn_learner(dls, resnet34, cbs=save_cbs)
    
    return learn

def get_last_model_file(model_file='geolife-21-cnn', dir_name='models'):
    
    file = sorted(Path(dir_name).glob(f'{model_file}*.pth'),
                  key = os.path.getmtime,
                  reverse=True)[0]
                           
    return file.stem

def test_setup(new=False): 
    
    if not new:
        df_val = load_pickle('observations_test_df.pkl')
    else:
        df = get_observations(DATA_PATH)
        df = df.reset_index()
        df_val = df[df.subset == 'val']
        save_pickle('observations_test_df.pkl', df_val)
        
    return df_val
def score_test(learn, df_val):
    dl_t = learn.dls.test_dl(df_val, bs=128)
    preds, _ = learn.get_preds(dl=dl_t)
    
    preds_ids = array([learn.dls.vocab[pred] for pred in predict_top_k_set(preds, 30)])
    
    y_val = df_val.species_id.astype(int)
    score_val = top_k_error_rate_from_sets(y_val, preds_ids)
    print("Top-30 error rate: {:.1%}".format(score_val))
df = get_observations(DATA_PATH)
df = df.reset_index()
df.head()
test_run = False
if test_run:
    df = df.sample(n=1_300_000, random_state = 1)
    df = df.reset_index(drop=True)
not_train_labels = set(df[df.subset == 'val'].species_id).difference(df[df.subset == 'train'].species_id)
len(not_train_labels)
cond = df.species_id.isin(not_train_labels)
remove_idx = np.where(cond)[0]
df = df.drop(index=remove_idx)
df = df.reset_index(drop=True)
dblock = DataBlock((ImageBlock, CategoryBlock),
                   get_x = get_x,
                   get_y = get_y,
                   splitter=splitter)
#dsets = dblock.datasets(df)
#dsets.train[0]
#dsets.valid[0]
df = load_pickle('observations_train_df.pkl')
dls = load_pickle('dataloaders.pkl')
dls = dblock.dataloaders(df, bs=256)
dls.show_batch(nrows=1, ncols=3)
save_pickle('observations_train_df.pkl', df)
save_pickle('dataloaders.pkl', dls)
#%time failed = verify_images(df.apply(get_x, axis=1))
#failed
#import csv

#with open('failed_imgs.csv', 'w') as f:
#    writer = csv.writer(f)
#    writer.writerow(failed)

#interp = ClassificationInterpretation.from_learner(learn)
#interp.plot_top_losses(3, nrows=1)
#interp.print_classification_report()
learn = cnn_learner(dls, resnet34)
lr_min, lr_steep = learn.lr_find()
lr_steep
save_cbs = SaveModelCallback(every_epoch=True, with_opt=True, fname='geolife-21-cnn')
learn = learn.load('geolife-21-cnn_1')
learn = training_setup(load_model=False, load_dls=False, model_file_name='geolife-21-cnn-full')
df_val = test_setup()
score_test(learn, df_val)
lr_min, lr_steep = learn.lr_find()
lr_steep
learn.fit_one_cycle(1, lr_steep)
score_test(learn, df_val)
lr_min, lr_steep = learn.lr_find()
lr_min, lr_steep
learn.opt.frozen_idx
learn.model
learn.unfreeze()
learn.fit_one_cycle(1, lr_max=slice(1e-6, 1e-2))
score_test(learn, df_val)
df_test = get_test_observations(DATA_PATH)
#df_test = df_test.reset_index()
#df_test.observation_id = df_test.observation_id.astype(int)
df_test.head()
df_test_obs_id_mapping = pd.read_csv(BASE_PATH / "test_observation_ids_mapping.csv", sep=";")
dl_t = learn.dls.test_dl(df_test, bs=256)
preds, _ = learn.get_preds(dl=dl_t)
preds_ids = array([learn.dls.vocab[pred] for pred in predict_top_k_set(preds, 30)])

# Generate the submission file
generate_submission_file(SUBMISSION_PATH/"fastai_cnn_rbg.csv", df_test_obs_id_mapping["Id"], preds_ids)
!kaggle competitions submit -c geolifeclef-2021 -f {SUBMISSION_PATH/"fastai_cnn_rbg.csv"} -m "fastai cnn submission"<jupyter_output>100%|██████████████████████████████████████| 5.68M/5.68M [00:01<00:00, 3.97MB/s]
Successfully submitted to GeoLifeCLEF 2021 - LifeCLEF 2021 x FGVC8
{'fraction_non_alphanumeric': 0.1206729605422416, 'fraction_numerical': 0.013435003631082063, 'mean_word_length': 3.2991675338189386, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '218191', 'license': 'Apache-2.0', 'n_tokens_mistral': 3092, 'n_tokens_neox': 2968, 'n_words': 550, 'path': 'GeoLifeCLEF-2021-cnn-satellite-train.ipynb', 'repo_name': 'rausial/cuadernos_kaggle', 'size': '8262'}