<jupyter_start><jupyter_text>Define parameters<jupyter_code>DATA_DIR = "data/"

# Bert pre-trained model selected in the list: bert-base-uncased, 
# bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased,
# bert-base-multilingual-cased, bert-base-chinese.
BERT_MODEL = 'bert-base-cased'

# The name of the task to train.I'm going to name this 'yelp'.
TASK_NAME = 'Relation Extraction'

# The output directory where the fine-tuned model and checkpoints will be written.
OUTPUT_DIR = f'outputs/{TASK_NAME}/'

# The directory where the evaluation reports will be written to.
REPORTS_DIR = f'reports/{TASK_NAME}_evaluation_report/'

# This is where BERT will look for pre-trained models to load parameters from.
CACHE_DIR = 'cache/'

# The maximum total input sequence length after WordPiece tokenization.
# Sequences longer than this will be truncated, and sequences shorter than this will be padded.
MAX_SEQ_LENGTH = 128

TRAIN_BATCH_SIZE = 24
EVAL_BATCH_SIZE = 8
LEARNING_RATE = 1e-5
NUM_TRAIN_EPOCHS = 10
RANDOM_SEED = 42
GRADIENT_ACCUMULATION_STEPS = 1
WARMUP_PROPORTION = 0.1


CONFIG_NAME = "config.json"
WEIGHTS_NAME = "pytorch_model.bin"<jupyter_output><empty_output><jupyter_text>Create Directories, if doesn't exist<jupyter_code>REPORTS_DIR
os.listdir(REPORTS_DIR)
if os.path.exists(REPORTS_DIR) and os.listdir(REPORTS_DIR):
  REPORTS_DIR += f'/report_{len(os.listdir(REPORTS_DIR))}'
  os.makedirs(REPORTS_DIR)
  print("yay")
if not os.path.exists(REPORTS_DIR):
  os.makedirs(REPORTS_DIR)
  REPORTS_DIR += f'/report_{len(os.listdir(REPORTS_DIR))}'
  os.makedirs(REPORTS_DIR)
    
# if os.path.exists(OUTPUT_DIR) and os.listdir(OUTPUT_DIR):
#         raise ValueError("Output directory ({}) already exists and is not empty.".format(OUTPUT_DIR))
if not os.path.exists(OUTPUT_DIR):
  os.makedirs(OUTPUT_DIR)
processor = MultiClassificationProcessor()
train_examples = processor.get_train_examples(DATA_DIR)
train_examples_len = len(train_examples)
print(train_examples[0])
train_examples_len
label_list = processor.get_labels() # [0, 1] for binary classification
num_labels = len(label_list)
# num_labels, label_list
print(label_list)
print(num_labels)
num_train_optimization_steps = int(
    train_examples_len / TRAIN_BATCH_SIZE / GRADIENT_ACCUMULATION_STEPS) * NUM_TRAIN_EPOCHS
tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False)
train_examples_for_processing = [(example,  MAX_SEQ_LENGTH, tokenizer) for example in train_examples]<jupyter_output><empty_output><jupyter_text>Convert Inputs to features in parallel threads -- Below 2 cells are commented, as it is needed only one time. We will use cached file from second time onwards<jupyter_code>process_count = cpu_count() - 1
 with Pool(process_count) as p:
     train_features = list(tqdm_notebook(p.imap(convert_examples_to_features.convert_example_to_feature, train_examples_for_processing), total=train_examples_len))
 with open(DATA_DIR + "train_features.pkl", "wb") as f:
     pickle.dump(train_features, f)
with open(DATA_DIR + "train_features.pkl", "rb") as f:
    train_features = pickle.load(f)
 model = BertForSequenceClassification.from_pretrained(BERT_MODEL, cache_dir=CACHE_DIR, num_labels=num_labels)

#model = Biobert_fc()
x = model.to(device)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
    ]
# param_optimizer

optimizer = BertAdam(optimizer_grouped_parameters,
                     lr=LEARNING_RATE,
                     warmup=WARMUP_PROPORTION,
                     t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
for f in train_features[0:10]:
  print(f.label_id)
logger.info("***** Running training *****")
logger.info("  Num examples = %d", train_examples_len)
logger.info("  Batch size = %d", TRAIN_BATCH_SIZE)
logger.info("  Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([int(f.label_id) for f in train_features], dtype=torch.long)


# torch.tensor([f.label_id for f in train_features])
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=TRAIN_BATCH_SIZE)
model.train()
for _ in trange(int(NUM_TRAIN_EPOCHS), desc="Epoch"):
    tr_loss = 0
    nb_tr_examples, nb_tr_steps = 0, 0
    for step, batch in enumerate(tqdm_notebook(train_dataloader, desc="Iteration")):
        batch = tuple(t.to(device) for t in batch)
        input_ids, input_mask, segment_ids, label_ids = batch

#         logits = model(input_ids, segment_ids, input_mask, labels=None)
        logits = model(input_ids, segment_ids, input_mask)

        loss_fct = CrossEntropyLoss()
        loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))

        if GRADIENT_ACCUMULATION_STEPS > 1:
            loss = loss / GRADIENT_ACCUMULATION_STEPS

        loss.backward()
        print("\r%f" % loss, end='')
        
        tr_loss += loss.item()
        nb_tr_examples += input_ids.size(0)
        nb_tr_steps += 1
        if (step + 1) % GRADIENT_ACCUMULATION_STEPS == 0:
            optimizer.step()
            optimizer.zero_grad()
            global_step += 1
model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self

# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(OUTPUT_DIR, WEIGHTS_NAME)
output_config_file = os.path.join(OUTPUT_DIR, CONFIG_NAME)

torch.save(model_to_save.state_dict(), output_model_file)
torch.save(model, OUTPUT_DIR + 'best_mdel')
# model_to_save.config.to_json_file(output_config_file)
# tokenizer.save_vocabulary(OUTPUT_DIR)
 #from sklearn import metrics
 #metrics.f1_score(df["labels"], logits, average='micro')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10569105691056911, 'fraction_numerical': 0.006722951844903065, 'mean_word_length': 5.115678776290631, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '285503', 'license': 'Apache-2.0', 'n_tokens_mistral': 2310, 'n_tokens_neox': 2229, 'n_words': 488, 'path': 'project_re/val_experiments/BERT_RE_i2b2_Val.ipynb', 'repo_name': 'pnarsina/w266_final', 'size': '6396'}