<jupyter_start><jupyter_text>Validation<jupyter_code>numberModels    = 6
numberFragments = 1

probaUpperBorn = 0.8
probaLowerBorn = 0.2
modelPathList   = df.path.iloc[:numberModels].values
vmodelResultList = []

for modelPath in modelPathList:
    for fragment in range(numberFragments):
        print(f'ModelPath={modelPath} Fragment={fragment:02d}')
        mdl = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber).to(device)
        mdl.load_state_dict(torch.load(modelPath))
        mdl.eval()
        modelResult = assess(mdl, validLoader, device)
        print(f'score={f1_score(modelResult[0], modelResult[1])}')
        vmodelResultList.append((modelPath,) + modelResult)
vresult_df = pd.DataFrame(vmodelResultList, columns=['name', 'Truth', 'Predicted', 'loss', 'Proba', 'Path'])
vresults   = np.vstack(vresult_df.Proba.values)
truth       = vresult_df.Truth.iloc[0]
paths       = vresult_df.Path.iloc[0]
result_mean = vresults.mean(axis=0)
result_std  = vresults.std(axis=0)
predicted   = (result_mean > 0.5).astype('int')
print(len(truth))
print(f1_score(truth, predicted))
vtruth       = truth[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]
vpaths       = paths[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]
vresult_prob = result_mean[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]
vresult_std  = result_std[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]

vpredicted    = (vresult_prob > 0.5).astype('int')
print(len(vtruth))
print(f1_score(vtruth, vpredicted))
etruth       = truth[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]
epaths       = paths[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]
eresult_prob = result_mean[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]
eresult_std  = result_std[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]
epredicted    = (eresult_prob > 0.5).astype('int')
print(len(etruth))
print(f1_score(etruth, epredicted))
errorDataset  = SampleDataset(epaths, etruth, sequenceSize, featureName)
errorLoader  = DataLoader(errorDataset,  batch_size=batchSize, shuffle=False, num_workers=numWorkers)

modelPathList   = df.path.iloc[:numberModels].values
emodelResultList = []

for modelPath in modelPathList:
    for fragment in range(numberFragments):
        print(f'ModelPath={modelPath} Fragment={fragment:02d}')
        mdl = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber).to(device)
        mdl.load_state_dict(torch.load(modelPath))
        mdl.eval()
        modelResult = assess(mdl, errorLoader, device)
        print(f'score={f1_score(modelResult[0], modelResult[1])}')
        emodelResultList.append((modelPath,) + modelResult)
eresult_df = pd.DataFrame(emodelResultList, columns=['name', 'Truth', 'Predicted', 'loss', 'Proba', 'Path'])
eresults   = np.vstack(eresult_df.Proba.values)

eresult_mean = eresults.mean(axis=0)
eresult_std  = eresults.std(axis=0)
epredicted   = (eresult_mean > 0.5).astype('int')
print(len(etruth))
print(f1_score(etruth, epredicted))
ftruth     = np.concatenate([vtruth, etruth])
fpredicted = np.concatenate([vpredicted, epredicted])
print(f1_score(ftruth, fpredicted))<jupyter_output>0.9655891553701773<jupyter_text>Testing<jupyter_code>modelPathList   = df.path.iloc[:numberModels].values
tmodelResultList = []

for modelPath in modelPathList:
    for fragment in range(numberFragments):
        print(f'ModelPath={modelPath} Fragment={fragment:02d}')
        mdl = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber).to(device)
        mdl.load_state_dict(torch.load(modelPath))
        mdl.eval()
        modelResult = assess(mdl, testLoader, device)
        print(f'score={f1_score(modelResult[0], modelResult[1])}')
        tmodelResultList.append((modelPath,) + modelResult)
tresult_df = pd.DataFrame(tmodelResultList, columns=['name', 'Truth', 'Predicted', 'loss', 'Proba', 'Path'])
tresults   = np.vstack(tresult_df.Proba.values)
tresult_df.head()
truth       = tresult_df.Truth.iloc[0]
paths       = tresult_df.Path.iloc[0]
result_mean = tresults.mean(axis=0)
result_std  = tresults.std(axis=0)
predicted   = (result_mean > 0.5).astype('int')
print(len(truth))
print(f1_score(truth, predicted))
vtruth       = truth[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]
vpaths       = paths[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]
vresult_prob = result_mean[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]
vresult_std  = result_std[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]

vpredicted    = (vresult_prob > 0.5).astype('int')
print(len(vtruth))
print(f1_score(vtruth, vpredicted))
etruth       = truth[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]
epaths       = paths[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]
eresult_prob = result_mean[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]
eresult_std  = result_std[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]
epredicted    = (eresult_prob > 0.5).astype('int')
print(len(etruth))
print(f1_score(etruth, epredicted))
errorDataset  = SampleDataset(epaths, etruth, sequenceSize, featureName)
errorLoader  = DataLoader(errorDataset,  batch_size=batchSize, shuffle=False, num_workers=numWorkers)

modelPathList   = df.path.iloc[:numberModels].values
emodelResultList = []

for modelPath in modelPathList:
    for fragment in range(numberFragments):
        print(f'ModelPath={modelPath} Fragment={fragment:02d}')
        mdl = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber).to(device)
        mdl.load_state_dict(torch.load(modelPath))
        mdl.eval()
        modelResult = assess(mdl, errorLoader, device)
        print(f'score={f1_score(modelResult[0], modelResult[1])}')
        emodelResultList.append((modelPath,) + modelResult)
eresult_df = pd.DataFrame(emodelResultList, columns=['name', 'Truth', 'Predicted', 'loss', 'Proba', 'Path'])
eresults   = np.vstack(eresult_df.Proba.values)

eresult_mean = eresults.mean(axis=0)
eresult_std  = eresults.std(axis=0)
epredicted   = (eresult_mean > 0.5).astype('int')
print(len(etruth))
print(f1_score(etruth, epredicted))
ftruth     = np.concatenate([vtruth, etruth])
fpredicted = np.concatenate([vpredicted, epredicted])
print(f1_score(ftruth, fpredicted))<jupyter_output>0.9635667531184389
{'fraction_non_alphanumeric': 0.1362796427471512, 'fraction_numerical': 0.015090853095164768, 'mean_word_length': 5.758584807492196, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '574373', 'license': 'MIT', 'n_tokens_mistral': 2362, 'n_tokens_neox': 2255, 'n_words': 352, 'path': 'zz_fragement_analysis.ipynb', 'repo_name': 'Mouatez/PetaDroid', 'size': '6494'}