<jupyter_start><jupyter_text>Dataset Exploration<jupyter_code>df = pd.read_csv('./data/twtc.csv')

df_label_mask = (df.drop(columns='label') == 0).assign(label=False)
df = df.mask(df_label_mask)

df.describe().T[['mean', 'std', 'min', '50%', 'max']]
pip install datasets
import pandas as pd
from datasets import Dataset
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from transformers import TrainingArguments, Trainer
from transformers import BertForSequenceClassification, BertTokenizer
labeled = pd.read_csv('data/labeled_scouting.csv')
unlabeled = pd.read_csv('data/unlabeled_scouting.csv')
test = pd.read_csv('data/augment/test.csv')
train = pd.concat([labeled, test]).drop_duplicates(keep=False)
labeled
print(len(labeled['text'] == 1))
print(len(unlabeled))

# encoded_corpus = tokenizer(text=train['text'].tolist(),
#                             add_special_tokens=True,
#                             padding='max_length',
#                             truncation='longest_first',
#                             max_length=300,
#                             return_attention_mask=True)

def tokenize_data(df, tokenizer):
    tokenized_df = tokenizer(df['text'].tolist(), 
                                add_special_tokens=True,
                                truncation=True, 
                                padding="max_length",
                                return_attention_mask=True)

    input_ids = np.array(tokenized_df['input_ids'])
    attention_mask = np.array(tokenized_df['attention_mask'])
    return input_ids, attention_mask, np.array(df['label'])


tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
train_input_ids, train_attention_mask, train_labels = tokenize_data(train, tokenizer)
test_input_ids, test_attention_mask, test_labels = tokenize_data(test, tokenizer)
                                
print(train['text'].iloc[0])

print(train_input_ids[0])

X_train = np.random.randint(-1,3, size=(3, 3))
scaler = StandardScaler()
scaler.fit(X_train)
scaler.transform(X_train)
def compute_mixup(a, b, l):
    return l * np.array(a) + (1 - l) * np.array(b)


def mixup(row1, row2, l):
    mixed_val = list(map(int, compute_mixup(row1['input_ids'].iloc[0], row2['input_ids'].iloc[0], l)))
    mixed_label = compute_mixup(row1['label'], row2['label'], l)

    if sum(row1['attention_mask'].iloc[0]) > sum(row2['attention_mask'].iloc[0]):
        new_attention = row1['attention_mask'].iloc[0]
    else:
        new_attention = row2['attention_mask'].iloc[0]

    return {
        'label': mixed_label,
        'input_ids': mixed_val,
        'token_type_ids': row1['token_type_ids'].iloc[0],
        'attention_mask': new_attention
    }

def mixup_aug(input_ids, att_masks, labels, lam, n=5):
    mixed_input_ids = []
    mixed_att_masks = []
    mixed_labels = []
    
    for i in range(n):
        a = np.random.randint(len(input_ids))
        b = np.random.randint(len(input_ids))
        
        x = list(map(int, compute_mixup(input_ids[a], input_ids[b], lam)))
        print(x)
        
    
        rows.append(mixup(row1, row2, lam))
    tokenized_train = pd.DataFrame(rows)
    shuffle(tokenized_train)
    return Dataset.from_pandas(tokenized_train)
mixup_aug(train_input_ids, train_attention_mask, train_labels, 0.4)
import torch
from torch.utils.data import TensorDataset, DataLoader

batch_size = 16

def create_dataloaders(inputs, masks, labels, batch_size):
    input_tensor = torch.tensor(inputs)
    mask_tensor = torch.tensor(masks)
    labels_tensor = torch.tensor(labels)
    dataset = TensorDataset(input_tensor, mask_tensor, 
                            labels_tensor)
    dataloader = DataLoader(dataset, batch_size=batch_size, 
                            shuffle=True)
    return dataloader

train_dataloader = create_dataloaders(train_input_ids, train_attention_mask, 
                                      train_labels, batch_size)
test_dataloader = create_dataloaders(test_input_ids, test_attention_mask, 
                                     test_labels, batch_size)
mixup_aug(train, 0.5)
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)

arguments = TrainingArguments(
    output_dir="sample_SD_trainer",
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    evaluation_strategy="epoch", # run validation at the end of each epoch
    save_strategy="epoch",
    learning_rate=2e-5,
    load_best_model_at_end=True,
    seed=224
)


def compute_metrics(eval_pred):
    """Called at the end of validation. Gives accuracy"""
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    # calculates the accuracy
    return {"accuracy": np.mean(predictions == labels)}


trainer = Trainer(
    model=model,
    args=arguments,
    train_dataset=train,
    eval_dataset=test, # change to test when you do your final evaluation!
    tokenizer=tokenizer,
    compute_metrics=compute_metrics
)
from transformers import TrainerCallback, EarlyStoppingCallback

class LoggingCallback(TrainerCallback):
    def __init__(self, log_path):
        self.log_path = log_path
        
    def on_log(self, args, state, control, logs=None, **kwargs):
        _ = logs.pop("total_flos", None)
        if state.is_local_process_zero:
            with open(self.log_path, "a") as f:
                f.write(json.dumps(logs) + "\n")


trainer.add_callback(EarlyStoppingCallback(early_stopping_patience=1, early_stopping_threshold=0.0))
trainer.add_callback(LoggingCallback("sample_SD_trainer/log.jsonl"))
trainer.train()<jupyter_output>/Users/amolsingh/anaconda3/envs/local_nmt/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use thePyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  warnings.warn(
***** Running training *****
  Num examples = 6222
  Num Epochs = 3
  Instantaneous batch size per device = 16
  Total train batch size (w. parallel, distributed & accumulation) = 16
  Gradient Accumulation steps = 1
  Total optimization steps = 1167<jupyter_text>From_Pretrained<jupyter_code># To load our saved model, we can pass the path to the checkpoint into the `from_pretrained` method:
test_str = "I enjoyed the movie!"

finetuned_model = AutoModelForSequenceClassification.from_pretrained("sample_SD_trainer/checkpoint-24")
model_inputs = tokenizer(test_str, return_tensors="pt")
prediction = torch.argmax(finetuned_model(**model_inputs).logits)
print(["NEGATIVE", "POSITIVE"][prediction])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11057979677226538, 'fraction_numerical': 0.011655708308427973, 'mean_word_length': 3.6804195804195805, 'pattern_counts': {'":': 1, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '471690', 'license': 'MIT', 'n_tokens_mistral': 2224, 'n_tokens_neox': 2137, 'n_words': 404, 'path': 'mixup.ipynb', 'repo_name': 'eishm721/cs224n-final-proj', 'size': '6692'}