<jupyter_start><jupyter_text>Model parameters<jupyter_code># Model parameters
BATCH_SIZE = 128
HEIGHT = 156
WIDTH = 156
CANAL = 3
N_CLASSES = labels.shape[0]
train_datagen=ImageDataGenerator(rescale=1./255)

train_generator=train_datagen.flow_from_dataframe(
    dataframe=train,
    directory="../input/imet-2019-fgvc6/train",
    x_col="id",
    y_col="attribute_ids",
    batch_size=BATCH_SIZE,
    shuffle=True,
    class_mode="categorical",
    target_size=(HEIGHT, WIDTH),
    subset='training')

test_datagen = ImageDataGenerator(rescale=1./255)

test_generator = test_datagen.flow_from_dataframe(  
        dataframe=test,
        directory = "../input/imet-2019-fgvc6/test",    
        x_col="id",
        target_size=(HEIGHT, WIDTH),
        batch_size=1,
        shuffle=False,
        class_mode=None)<jupyter_output>Found 109237 images belonging to 1103 classes.
Found 7443 images.<jupyter_text>Model<jupyter_code>def create_model(input_shape, n_out):
    input_tensor = Input(shape=input_shape)
    base_model = applications.Xception(weights=None, include_top=False,
                   input_tensor=input_tensor)

    x = GlobalAveragePooling2D()(base_model.output)
    x = Dropout(0.5)(x)
    final_output = Dense(n_out, activation='sigmoid', name='final_output')(x)
    model = Model(input_tensor, final_output)
    
    return model
weights_paths = ['../input/imet-xcecption-pretrained/Xcecption_fold1.h5', '../input/imet-xcecption-pretrained/Xcecption_fold2.h5', 
                 '../input/imet-xcecption-pretrained/Xcecption_fold3.h5', '../input/imet-xcecption-pretrained/Xcecption_fold4.h5', 
                 '../input/imet-xcecption-pretrained/Xcecption_fold5.h5']
model = create_model(input_shape=(HEIGHT, WIDTH, CANAL), n_out=N_CLASSES)
STEP_SIZE_TEST = test_generator.n//test_generator.batch_size
preds = np.empty((0, N_CLASSES))
n_folds = len(weights_paths)
threshold = 0.1<jupyter_output>WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.<jupyter_text>Apply model to test set and output predictions<jupyter_code>for weights_path in weights_paths:
    model.load_weights(weights_path)
    test_generator.reset()
    preds = model.predict_generator(test_generator, steps=STEP_SIZE_TEST)
preds = preds / n_folds
predictions = []
for pred_ar in preds:
    valid = []
    for idx, pred in enumerate(pred_ar):
        if pred > threshold:
            valid.append(idx)
    if len(valid) == 0:
        valid.append(np.argmax(pred_ar))
    predictions.append(valid)
filenames = test_generator.filenames
label_map = {train_generator.class_indices[k] : k for k in train_generator.class_indices}

results = pd.DataFrame({'id':filenames, 'attribute_ids':predictions})
results['id'] = results['id'].map(lambda x: str(x)[:-4])
results['attribute_ids'] = results['attribute_ids'].apply(lambda x: list(map(label_map.get, x)))
results["attribute_ids"] = results["attribute_ids"].apply(lambda x: ' '.join(x))
results.to_csv('submission.csv',index=False)
results.head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1218020022246941, 'fraction_numerical': 0.021134593993325918, 'mean_word_length': 4.7552, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '145559', 'license': 'MIT', 'n_tokens_mistral': 1275, 'n_tokens_neox': 1217, 'n_words': 211, 'path': 'Model backlog/Deep Learning/Xception/[205th] - Xception - K Fold blending 2.ipynb', 'repo_name': 'dimitreOliveira/iMet-Collection-2019-FGVC6', 'size': '3596'}