<jupyter_start><jupyter_text>Load Data<jupyter_code>features = submission_tools.read_all_features(
    '/scratch/utrerf/trigger-attack/trojai_submission/scratch/RHPP1CJR0B')
features.head()
mean_std = features.groupby(['location', 'source_label', 'target_label'], as_index=False).agg({'test_loss':['mean', 'std']})
columns = []
for i in mean_std.columns:
    result = i[0]
    if len(i[1]) > 0:
        result += f'_{i[1]}'
    columns.append(result)

mean_std.columns = columns

merged_df = pd.merge(features, mean_std, on=['location', 'source_label', 'target_label'], how='left')

merged_df['normalized_test_loss'] = (merged_df['test_loss'] - merged_df['test_loss_mean'])/merged_df['test_loss_std']
merged_df.head()<jupyter_output><empty_output><jupyter_text>Visualize Data<jupyter_code>idx = merged_df.groupby(['model_name'])['normalized_test_loss'].transform(min) == merged_df['normalized_test_loss']

min_features = merged_df[idx]
min_features['normalized_test_loss'] = min_features['normalized_test_loss'].astype(float)
min_features['task'] = min_features['task'].str.upper()


sns.boxplot(data=min_features, x='task', y='normalized_test_loss', hue='poisoned')
# plt.ylim(-.05, 1.)
plt.legend(loc='upper right', title='Poisoned')
plt.ylabel('Trigger Reconstruction Loss')
plt.xlabel('Task')
plt.savefig('normalization_hurts.jpeg', dpi=2000)
idx = merged_df.groupby(['model_name'])['test_loss'].transform(min) == merged_df['test_loss']

min_features = merged_df[idx]
min_features['test_loss'] = min_features['test_loss'].astype(float)
min_features['task'] = min_features['task'].str.upper()

sns.boxplot(data=min_features, x='task', y='test_loss', hue='poisoned')
plt.ylim(-.05, 1.)
plt.legend(loc='upper right', title='Poisoned')
plt.ylabel('Trigger Reconstruction Loss')
plt.xlabel('Task')
plt.savefig('round9_mar8.jpeg', dpi=2000)
import numpy as np

idx = merged_df.groupby(['model_name'])['test_loss'].transform(min) == merged_df['test_loss']

min_features = merged_df[idx]
min_features['test_loss'] = min_features['test_loss'].astype(float)
min_features['task'] = min_features['task'].str.upper()
min_features['log_loss'] = -np.log(min_features['test_loss'])
ixs = min_features['log_loss'] > 12
min_features['log_loss'].loc[ixs] = 12

sns.boxplot(data=min_features, x='task', y='log_loss', hue='poisoned')
plt.legend(loc='upper right', title='Poisoned')
plt.ylabel('Trigger Reconstruction Loss')
plt.xlabel('Task')
plt.savefig('log_loss.jpeg', dpi=2000)<jupyter_output><ipython-input-39-891c81060b81>:6: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  min_features['test_loss'] = min_features['test_loss'].astype(float)
<ipython-input-39-891c81060b81>:7: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  min_features['task'] = min_features['task'].str.upper()
<ipython-input-39-891c81060b81>:8: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http[...]<jupyter_text>Model<jupyter_code>from sklearn.preprocessing import OneHotEncoder

enc = OneHotEncoder(handle_unknown='ignore', sparse=False)
result = enc.fit_transform(min_features[['task']])
X = pd.DataFrame(data=result, columns=list(enc.categories_[0]))
X['loss'] = min_features['test_loss'].reset_index(drop=True)

y = min_features['poisoned'].reset_index(drop=True)
X.head()
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.calibration import CalibratedClassifierCV
from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier()
rsk = RepeatedStratifiedKFold(n_splits=5, n_repeats=20)
calibrated_clf = CalibratedClassifierCV(base_estimator=clf, cv=rsk)
calibrated_clf.fit(X, y)

# from joblib import dump, load
# dump(calibrated_clf, 'classifier.joblib') 
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss

num_iterations = 20
test_size = .15

acc_list, cross_entropy_list = [], []
for i in range(num_iterations):
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=test_size, random_state=i)

    clf = RandomForestClassifier()
    rsk = RepeatedStratifiedKFold(n_splits=5, n_repeats=20)
    calibrated_clf = CalibratedClassifierCV(base_estimator=clf, cv=rsk)
    calibrated_clf.fit(X_train, y_train)
    acc_list.append(calibrated_clf.score(X_test, y_test))
    cross_entropy_list.append(log_loss(y_test, calibrated_clf.predict_proba(X_test)))
fig, axs = plt.subplots(1, 2, figsize=(8, 3))
sns.boxplot(acc_list, ax=axs[0])
axs[0].set_xlabel('CV Accuracy %')
sns.boxplot(cross_entropy_list, ax=axs[1])
plt.xlabel('CV Cross-Entropy %')
plt.savefig('apr5_results.jpeg', bbox_inches="tight", dpi=2000)
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss


num_iterations = 20
test_size = .15

task_df = min_features[min_features['task']=='SC']
sc_acc_list, sc_cross_entropy_list = [], []
for i in range(num_iterations):
    X_train, X_test, y_train, y_test = train_test_split(
        task_df[['loss']], task_df['poisoned'], test_size=test_size, random_state=i)

    clf = LogisticRegression().fit(X_train, y_train)
    sc_acc_list.append(clf.score(X_test, y_test))
    sc_cross_entropy_list.append(log_loss(y_test, clf.predict_proba(X_test)))

task_df = min_features[min_features['task']=='NER']
ner_acc_list, ner_cross_entropy_list = [], []
for i in range(num_iterations):
    X_train, X_test, y_train, y_test = train_test_split(
        task_df[['loss']], task_df['poisoned'], test_size=test_size, random_state=i)

    clf = LogisticRegression().fit(X_train, y_train)
    ner_acc_list.append(clf.score(X_test, y_test))
    ner_cross_entropy_list.append(log_loss(y_test, clf.predict_proba(X_test)))

task_df = min_features[min_features['task']=='QA']
qa_acc_list, qa_cross_entropy_list = [], []
for i in range(num_iterations):
    X_train, X_test, y_train, y_test = train_test_split(
        task_df[['loss']], task_df['poisoned'], test_size=test_size, random_state=i)

    clf = LogisticRegression().fit(X_train, y_train)
    qa_acc_list.append(clf.score(X_test, y_test))
    qa_cross_entropy_list.append(log_loss(y_test, clf.predict_proba(X_test)))
all_dfs = []

temp_df = pd.DataFrame([])
temp_df['accuracy'] = qa_acc_list
temp_df['task'] = 'QA'
all_dfs.append(temp_df)

temp_df = pd.DataFrame([])
temp_df['accuracy'] = ner_acc_list
temp_df['task'] = 'NER'
all_dfs.append(temp_df)

temp_df = pd.DataFrame([])
temp_df['accuracy'] = sc_acc_list
temp_df['task'] = 'SC'
all_dfs.append(temp_df)

acc_df = pd.concat(all_dfs)

sns.boxplot(data=acc_df, x='accuracy', y='task')
plt.xlabel('CV Accuracy %')
plt.ylabel('Task')
plt.savefig('CV_Accuracy.jpeg', dpi=2000)
acc_df
x = min_features[(min_features['task']=='QA') & (min_features['poisoned']==True)].sort_values('loss')
len(x)<jupyter_output><empty_output><jupyter_text>DL<jupyter_code>import torch
x = torch.arange(12, dtype=torch.float32)
x
x.shape
X = x.reshape((3,4))
X
torch.zeros((3,4))
torch.ones((3,4))
torch.randn((3,4))
x = torch.tensor([1.0, 2, 4, 8])
y = torch.tensor([2, 2, 2, 2])
x+y,x-y,x*y,x/y,x**y
X[1:]
A = X.numpy()
B = torch.from_numpy(A)
type(A), type(B)
import os
os.makedirs(os.path.join('..', 'data'), exist_ok=True)
data_file = os.path.join('..', 'data', 'house_tiny.csv')
with open(data_file, 'w') as f:
    f.write('NumRooms,Alley,Price\n')  # Column names
    f.write('NA,Pave,127500\n')  # Each row represents a data example
    f.write('2,NA,106000\n')
    f.write('4,NA,178100\n')
    f.write('NA,NA,140000\n')
import pandas as pd
data = pd.read_csv(data_file)
print(data)
inputs, outputs = data.iloc[:, 0:2], data.iloc[:, 2]
inputs
 inputs = pd.get_dummies(inputs, dummy_na=True)
 inputs
X, y = torch.tensor(inputs.values), torch.tensor(outputs.values)
X, y
inputs.values
X = torch.ones((3,4,5))
X.sum(dim=[0,1])
X.numel()
X.norm()
torch.sqrt(3*(X[1].norm()**2))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15776499589153656, 'fraction_numerical': 0.018664162460382674, 'mean_word_length': 7.045325779036827, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '554948', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3434, 'n_tokens_neox': 3281, 'n_words': 531, 'path': 'trojai_submission/results.ipynb', 'repo_name': 'trigger-attack/trigger-attack', 'size': '8519'}