<jupyter_start><jupyter_text>Positive Set<jupyter_code>positive_df=pd.merge(novel_compounds,head_features,on=["head"])
positive_df=pd.merge(positive_df,modifier_features,on=["modifier"])
positive_df['Plausibility']=True
positive_df<jupyter_output><empty_output><jupyter_text>Negative Sets<jupyter_code>def neg_df_creator(file):
    pkl_file=pkl.load( open(file,'rb'))
    df=pd.DataFrame(pkl_file)
    df.columns=['modifier','head']
    negative_df=pd.merge(df,head_features,on=["head"])
    negative_df=pd.merge(negative_df,modifier_features,on=["modifier"])
    negative_df['Plausibility']=False
    return negative_df
def df_joiner(files):
    df_list=[]
    for file in files:
        neg_df=neg_df_creator(file)
        whole_df=pd.concat([neg_df,positive_df])
        df_list.append(whole_df)
    
    return df_list
corrupt_modifier_files=[]
for file in glob.glob("/data/dharp/compounding/datasets/corrupt_modifier*"):
    corrupt_modifier_files.append(file)
corrupt_modifiers=df_joiner(corrupt_modifier_files)
corrupt_head_files=[]
for file in glob.glob("/data/dharp/compounding/datasets/corrupt_head*"):
    corrupt_head_files.append(file)
corrupt_heads=df_joiner(corrupt_head_files)
acc_ch=[]
for i,corrupt_head in enumerate(corrupt_heads):
    data=corrupt_head.drop(['modifier','head'],axis=1)
    X, Y = data.iloc[:,:-1],data.iloc[:,-1]
    
    X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=1991)
    # fit model no training data
    model = XGBClassifier(njobs=-1)
    model.fit(X_train, y_train)
    # make predictions for test data
    y_pred = model.predict(X_test)
    predictions = [round(value) for value in y_pred]
    # evaluate predictions
    print("DF",i+1)
    accuracy = accuracy_score(y_test, predictions)
    acc_ch.append(accuracy)
    print("Accuracy: %.2f%%" % (accuracy * 100.0))

round(np.mean(acc_ch)*100,2)
round(np.std(acc_ch)*100,2)
acc_cm=[]
for i,corrupt_head in enumerate(corrupt_modifiers):
    data=corrupt_head.drop(['modifier','head'],axis=1)
    X, Y = data.iloc[:,:-1],data.iloc[:,-1]
    X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=1991)
# fit model no training data
    model = XGBClassifier(njobs=-1)
    model.fit(X_train, y_train)
# make predictions for test data
    y_pred = model.predict(X_test)
    predictions = [round(value) for value in y_pred]
# evaluate predictions
    accuracy = accuracy_score(y_test, predictions)
    acc_cm.append(accuracy)
    print("DF",i+1)
    print("Accuracy: %.2f%%" % (accuracy * 100.0))
round(np.mean(acc_cm)*100,2)
round(np.std(acc_cm)*100,2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14323507180650039, 'fraction_numerical': 0.018140589569160998, 'mean_word_length': 5.58457711442786, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '286097', 'license': 'Apache-2.0', 'n_tokens_mistral': 1036, 'n_tokens_neox': 1005, 'n_words': 146, 'path': 'novel_compound_predictor/DFM_DecadeAgnostic.ipynb', 'repo_name': 'prajitdhar/Compositionality', 'size': '2646'}