<jupyter_start><jupyter_text>Inference on images training set<jupyter_code>model = keras.models.load_model('best_model.h5')
model.summary()
preds = model.predict(test_gen, use_multiprocessing=False, workers=4, verbose=1)
print(f'preds.shape={preds.shape}')
preds = preds.T
data['fvc_last_3_cv'] = preds[0]
data['fvc_last_2_cv'] = preds[1]
data['fvc_last_1_cv'] = preds[2]
data.info()
data.head()
data.drop(['img'], axis=1, inplace=True)
def set_dist(row, fvc_last_1_cv, fvc_last_2_cv, fvc_last_3_cv, lung_area, tissue_area, lung_tissue_ratio) -> None:
    quantiles = [0, 0.5, 0.75, 0.9, 0.95, 0.99, 1]
    s = pd.Series(fvc_last_1_cv)
    qs = s.quantile(quantiles).to_numpy()
    row['fvc_last_1_min'] = qs[0]
    row['fvc_last_1_p50'] = qs[1]
    row['fvc_last_1_p75'] = qs[2]
    row['fvc_last_1_p90'] = qs[3]
    row['fvc_last_1_p95'] = qs[4]
    row['fvc_last_1_p99'] = qs[5]
    row['fvc_last_1_max'] = qs[6]
    s = pd.Series(fvc_last_2_cv)
    qs = s.quantile(quantiles).to_numpy()
    row['fvc_last_2_min'] = qs[0]
    row['fvc_last_2_p50'] = qs[1]
    row['fvc_last_2_p75'] = qs[2]
    row['fvc_last_2_p90'] = qs[3]
    row['fvc_last_2_p95'] = qs[4]
    row['fvc_last_2_p99'] = qs[5]
    row['fvc_last_2_max'] = qs[6]
    s = pd.Series(fvc_last_3_cv)
    qs = s.quantile(quantiles).to_numpy()
    row['fvc_last_3_min'] = qs[0]
    row['fvc_last_3_p50'] = qs[1]
    row['fvc_last_3_p75'] = qs[2]
    row['fvc_last_3_p90'] = qs[3]
    row['fvc_last_3_p95'] = qs[4]
    row['fvc_last_3_p99'] = qs[5]
    row['fvc_last_3_max'] = qs[6]
    s = pd.Series(lung_area)
    qs = s.quantile(quantiles).to_numpy()
    row['lung_area_min'] = qs[0]
    row['lung_area_p50'] = qs[1]
    row['lung_area_p75'] = qs[2]
    row['lung_area_p90'] = qs[3]
    row['lung_area_p95'] = qs[4]
    row['lung_area_p99'] = qs[5]
    row['lung_area_max'] = qs[6]
    s = pd.Series(tissue_area)
    qs = s.quantile(quantiles).to_numpy()
    row['tissue_area_min'] = qs[0]
    row['tissue_area_p50'] = qs[1]
    row['tissue_area_p75'] = qs[2]
    row['tissue_area_p90'] = qs[3]
    row['tissue_area_p95'] = qs[4]
    row['tissue_area_p99'] = qs[5]
    row['tissue_area_max'] = qs[6]
    s = pd.Series(lung_tissue_ratio)
    qs = s.quantile(quantiles).to_numpy()
    row['lung_tissue_ratio_min'] = qs[0]
    row['lung_tissue_ratio_p50'] = qs[1]
    row['lung_tissue_ratio_p75'] = qs[2]
    row['lung_tissue_ratio_p90'] = qs[3]
    row['lung_tissue_ratio_p95'] = qs[4]
    row['lung_tissue_ratio_p99'] = qs[5]
    row['lung_tissue_ratio_max'] = qs[6]



rows = []
row: Dict[str, Any] = {}
prev = None
fvc_last_1_cv = []
fvc_last_2_cv = []
fvc_last_3_cv = []
lung_area = []
tissue_area = []
lung_tissue_ratio = []
for t in data.itertuples():
    # new patient
    if prev is not None and prev != t.pid:
        set_dist(row, fvc_last_1_cv, fvc_last_2_cv, fvc_last_3_cv, lung_area, tissue_area, lung_tissue_ratio)
        rows.append(row)
    if prev is None or prev != t.pid:
        row = {}
        fvc_last_1_cv = []
        fvc_last_2_cv = []
        fvc_last_3_cv = []
        lung_area = []
        tissue_area = []
        lung_tissue_ratio = []
        row['pid'] = t.pid
        row['age'] = t.age
        row['sex'] = t.sex
        row['smoking'] = t.smoking
        row['week_1'] = t.week_1
        row['fvc_1'] = t.fvc_1
        row['percent_1'] = t.percent_1
        row['fvc_last_1'] = t.fvc_last_1
        row['fvc_last_2'] = t.fvc_last_2
        row['fvc_last_3'] = t.fvc_last_3
        row['week_last_1'] = t.week_last_1
        row['week_last_2'] = t.week_last_2
        row['week_last_3'] = t.week_last_3
    prev = t.pid
    fvc_last_1_cv.append(t.fvc_last_1_cv)
    fvc_last_2_cv.append(t.fvc_last_2_cv)
    fvc_last_3_cv.append(t.fvc_last_3_cv)
    lung_area.append(t.lung_area)
    tissue_area.append(t.tissue_area)
    lung_tissue_ratio.append(t.lung_tissue_ratio)
    
# add the last patient!
if len(row) != 0:
    set_dist(row, fvc_last_1_cv, fvc_last_2_cv, fvc_last_3_cv, lung_area, tissue_area, lung_tissue_ratio)
    rows.append(row)



train = pd.DataFrame.from_records(rows)
assert len(train) == 174
train = train.astype({
    'pid': str,
    'age': np.uint8,
    'sex': str,
    'smoking': str,
    'week_1': np.int16,
    'fvc_1': np.uint16,
    'percent_1': np.float32,
    'fvc_last_1': np.uint16,
    'fvc_last_2': np.uint16,
    'fvc_last_3': np.uint16,
    'week_last_1': np.int16,
    'week_last_2': np.int16,
    'week_last_3': np.int16,
    'fvc_last_1_min': np.float32,
    'fvc_last_1_p50': np.float32,
    'fvc_last_1_p75': np.float32,
    'fvc_last_1_p90': np.float32,
    'fvc_last_1_p95': np.float32,
    'fvc_last_1_p99': np.float32,
    'fvc_last_1_max': np.float32,
    'fvc_last_2_min': np.float32,
    'fvc_last_2_p50': np.float32,
    'fvc_last_2_p75': np.float32,
    'fvc_last_2_p90': np.float32,
    'fvc_last_2_p95': np.float32,
    'fvc_last_2_p99': np.float32,
    'fvc_last_2_max': np.float32,
    'fvc_last_3_min': np.float32,
    'fvc_last_3_p50': np.float32,
    'fvc_last_3_p75': np.float32,
    'fvc_last_3_p90': np.float32,
    'fvc_last_3_p95': np.float32,
    'fvc_last_3_p99': np.float32,
    'fvc_last_3_max': np.float32,
    'lung_area_min': np.float32,
    'lung_area_p50': np.float32,
    'lung_area_p75': np.float32,
    'lung_area_p90': np.float32,
    'lung_area_p95': np.float32,
    'lung_area_p99': np.float32,
    'lung_area_max': np.float32,
    'tissue_area_min': np.float32,
    'tissue_area_p50': np.float32,
    'tissue_area_p75': np.float32,
    'tissue_area_p90': np.float32,
    'tissue_area_p95': np.float32,
    'tissue_area_p99': np.float32,
    'tissue_area_max': np.float32,
    'lung_tissue_ratio_min': np.float32,
    'lung_tissue_ratio_p50': np.float32,
    'lung_tissue_ratio_p75': np.float32,
    'lung_tissue_ratio_p90': np.float32,
    'lung_tissue_ratio_p95': np.float32,
    'lung_tissue_ratio_p99': np.float32,
    'lung_tissue_ratio_max': np.float32,
})
train.info()
train.head()
train.to_parquet('output/cvtrain.parquet', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15930521091811414, 'fraction_numerical': 0.06401985111662531, 'mean_word_length': 3.701399688958009, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '595668', 'license': 'MIT', 'n_tokens_mistral': 2976, 'n_tokens_neox': 2834, 'n_words': 369, 'path': 'efficientnet.ipynb', 'repo_name': 'seahrh/kaggle-osic-pulmonary-fibrosis-progression', 'size': '6045'}