<jupyter_start><jupyter_text>Entity Dataset, Entity Model & Training Function<jupyter_code>class EntityDataset:
    def __init__(self, texts, tags):
        # texts: [["hi", ",", "my", "name", "is", "abhishek"], ["hello".....]]
        # pos/tags: [[1 2 3 4 1 5], [....].....]]
        self.texts = texts
        #self.pos = pos
        self.tags = tags
    
    def __len__(self):
        return len(self.texts)
    
    def __getitem__(self, item):
        text = self.texts[item]
        #pos = self.pos[item]
        tags = self.tags[item]

        ids = []
        #target_pos = []
        target_tag =[]

        for i, s in enumerate(text):
            inputs = TOKENIZER.encode(
                s,
                add_special_tokens=False
            )
            # abhishek: ab ##hi ##sh ##ek
            input_len = len(inputs)
            ids.extend(inputs)
            #target_pos.extend([pos[i]] * input_len)
            target_tag.extend([tags[i]] * input_len)

        ids = ids[:MAX_LEN - 2]
        # for CLS/SEP tokens
        #target_pos = target_pos[:config.MAX_LEN - 2]
        target_tag = target_tag[:MAX_LEN - 2]

        ids = [2] + ids + [3]
        #target_pos = [0] + target_pos + [0]
        target_tag = [0] + target_tag + [0]

        mask = [1] * len(ids)
        token_type_ids = [0] * len(ids)

        padding_len = MAX_LEN - len(ids)

        ids = ids + ([0] * padding_len)
        mask = mask + ([0] * padding_len)
        token_type_ids = token_type_ids + ([0] * padding_len)
        #target_pos = target_pos + ([0] * padding_len)
        target_tag = target_tag + ([0] * padding_len)

        return {
            "ids": torch.tensor(ids, dtype=torch.long),
            "mask": torch.tensor(mask, dtype=torch.long),
            "token_type_ids": torch.tensor(token_type_ids, dtype=torch.long),
            "target_tag": torch.tensor(target_tag, dtype=torch.long),
        }
def train_fn(data_loader, model, optimizer, device, scheduler):
    model.train()
    final_loss = 0
    for data in tqdm(data_loader, total=len(data_loader)):
        for k, v in data.items():
            data[k] = v.to(device)
        optimizer.zero_grad()
        _, loss = model(**data)
        loss.backward()
        optimizer.step()
        scheduler.step()
        final_loss += loss.item()
    return final_loss / len(data_loader)


def eval_fn(data_loader, model, device):
    model.eval()
    final_loss = 0
    for data in tqdm(data_loader, total=len(data_loader)):
        for k, v in data.items():
            data[k] = v.to(device)
        _, loss = model(**data)
        final_loss += loss.item()
    return final_loss / len(data_loader)
def loss_fn(output, target, mask, num_labels):
    lfn = nn.CrossEntropyLoss()
    active_loss = mask.view(-1) == 1
    active_logits = output.view(-1, num_labels)
    active_labels = torch.where(
        active_loss,
        target.view(-1),
        torch.tensor(lfn.ignore_index).type_as(target)
    )
    loss = lfn(active_logits, active_labels)
    return loss


class EntityModel(nn.Module):
    def __init__(self, num_tag):
        super(EntityModel, self).__init__()
        self.num_tag = num_tag
        self.bert = transformers.BertModel.from_pretrained(BASE_MODEL_PATH,return_dict=False)
        self.bert_drop_1 = nn.Dropout(0.3)
        #self.bert_drop_2 = nn.Dropout(0.3)
        self.out_tag = nn.Linear(768, self.num_tag)

    
    def forward(self, ids, mask, token_type_ids, target_tag):
        o1, _ = self.bert(ids, attention_mask=mask, token_type_ids=token_type_ids)

        bo_tag = self.bert_drop_1(o1)
        #bo_pos = self.bert_drop_2(o1)

        tag = self.out_tag(bo_tag)
        #pos = self.out_pos(bo_pos)

        loss_tag = loss_fn(tag, target_tag, mask, self.num_tag)
        #loss_pos = loss_fn(pos, target_pos, mask, self.num_pos)

        #loss = (loss_tag + loss_pos) / 2
        loss = loss_tag

        return tag, loss

def process_data(data_path):
    df = pd.read_csv(data_path, encoding="latin-1")
    df=df.replace(r'^\s*$', np.nan, regex=True)
    df.loc[:, "Sentence #"] = df["Sentence #"].fillna(method="ffill")
    print('Number of empty values are ', df["Word"].isna().sum())
    df["Word"].fillna("None", inplace = True)
    df.loc[:, "Tag"] = df["Tag"].fillna(method="ffill")

    print("Check 1")
    enc_pos = preprocessing.LabelEncoder()
    enc_tag = preprocessing.LabelEncoder()
    print("Check 2")


    #df.loc[:, "POS"] = enc_pos.fit_transform(df["POS"])
    df.loc[:, "Tag"] = enc_tag.fit_transform(df["Tag"])
    print("Check 3")

    
    sentences = df.groupby("Sentence #")["Word"].apply(list).values
    print("Check 4")

    #pos = df.groupby("Sentence #")["POS"].apply(list).values
    pos = []
    tag = df.groupby("Sentence #")["Tag"].apply(list).values
    print("Check 5")

    return sentences, pos, tag, enc_pos, enc_tag<jupyter_output><empty_output><jupyter_text>Preprocess Dataset<jupyter_code>sentences, pos, tag, enc_pos, enc_tag = process_data(TRAINING_FILE)
print("\nsentences:\n", sentences)
print("\n pos:\n", pos)
print("\n tag:\n", tag)
print("\n enc_pos:\n", enc_pos)
print("\n enc_tag:\n", enc_tag)
meta_data = {
        "enc_pos": enc_pos,
        "enc_tag": enc_tag
    }
joblib.dump(meta_data, "meta.bin")
num_tag = len(list(enc_tag.classes_))
print(num_tag)
(train_sentences, test_sentences, train_tag,test_tag) = model_selection.train_test_split(sentences,  tag, random_state=42, test_size=0.2)
train_dataset = EntityDataset(
        texts=train_sentences, tags=train_tag
    )

train_data_loader = torch.utils.data.DataLoader(
        train_dataset, batch_size=TRAIN_BATCH_SIZE, num_workers=2
    )

valid_dataset = EntityDataset(
        texts=test_sentences, tags=test_tag
    )

valid_data_loader = torch.utils.data.DataLoader(
        valid_dataset, batch_size=VALID_BATCH_SIZE, num_workers=1
    )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = EntityModel(num_tag=num_tag)
model.to(device)<jupyter_output>Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight']
- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).<jupyter_text>Training<jupyter_code>#import os
#os.mkdir(MODEL_PATH)
#data
# for data in tqdm(train_data_loader, total=len(train_data_loader)):
#   print('Next Data Loader')
param_optimizer = list(model.named_parameters())
no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
        {
            "params": [
                p for n, p in param_optimizer if not any(nd in n for nd in no_decay)
            ],
            "weight_decay": 0.001,
        },
        {
            "params": [
                p for n, p in param_optimizer if any(nd in n for nd in no_decay)
            ],
            "weight_decay": 0.0,
        },
    ]

num_train_steps = int(len(train_sentences) / TRAIN_BATCH_SIZE * EPOCHS)
optimizer = AdamW(optimizer_parameters, lr=3e-5)
scheduler = get_linear_schedule_with_warmup(
        optimizer, num_warmup_steps=0, num_training_steps=num_train_steps
    )


best_loss = np.inf
for epoch in range(EPOCHS):
  train_loss = train_fn(train_data_loader, model, optimizer, device, scheduler)
  test_loss = eval_fn(valid_data_loader, model, device)
  print(f"Train Loss = {train_loss} Valid Loss = {test_loss}")
  if test_loss < best_loss:
    torch.save(model.state_dict(), MODEL_PATH)
    best_loss = test_loss<jupyter_output>100%|██████████| 25/25 [00:27<00:00,  1.11s/it]
100%|██████████| 26/26 [00:07<00:00,  3.60it/s]<jupyter_text>Prediction<jupyter_code>meta_data = joblib.load("meta.bin")
enc_tag = meta_data["enc_tag"]

#num_pos = len(list(enc_pos.classes_))
num_tag = len(list(enc_tag.classes_))
# sentence = "seasoned backend developer. entrepreneur. open source contributor. scalable, highly-available web development: python (django), ruby (rails, sinatra), node.js, go, react js/native, angular, java. web backend scalability and performance tuning: new relic, ruby-prof, cprofile. queue-based solutions: kue, resque/sidekiq, celery, jms, rabbitmq. mobile: swift, objective-c, restkit/afnetworking, coredata, corelocation, gcd, sentestingkit, android studio, play service, retrofit. deep learning: convolutional neural network. test-driven-development: rspec, cucumber, python unittest, junit, jasmine. continuous integration/delivery: travis, jenkins, capistrano, vagrant, git, subversion, rake, maven, ant, buildout, make/gnu make. cloud/container: aws, azure, docker, docker-compose, dcos, kubernetes. devops: mesos, chef, puppet, mcollective, pxe, ipmi, nagios, zabbix. scripting: bash, python, ruby, perl. open source projects that enjoy 200+ stars on github and 100+k downloads on sourceforge., seasoned backend developer and entrepreneur.\n\nspecialties: \nscalable, highly-available web development: java, ruby (rails, sinatra), python (django), node.js.\nqueue-based solutions: resque/sidekiq, celery, jms, rabbitmq.\nweb backend performance tuning.\nios: coredata, corelocation, gcd, restkit/afnetworking, sentestingkit, swift.\ntest-driven-development: rspec, cucumber, python unittest, junit.\ncontinuous integration/delivery: jenkins, chef, capistrano, vagrant, git, subversion, rake, maven, ant, buildout, make/gnu make.\ninfrastructure-as-a-service: openstack nova, aws ec2.\nplatform-as-a-service: cloudfoundry, heroku, rightscale, enstratus, scalr, juju.\ncloud storage: hadoop hdfs, aws s3, openstack swift, mongodb.\nagile methodologies: scrum, fdd (feature-driven-development).\ndevops: chef, puppet, mcollective, pxe, ipmi, nagios, zabbix.\nscripting: bash, python, ruby, perl., programmer. entrepreneur at banian labs, pdh - networking/network engineering, pdh - network planner/provisioning, vp engineering at rhumbix"
# tokenized_sentence = TOKENIZER.encode(sentence)

# sentence = sentence.split()
# print(sentence)
# print(tokenized_sentence)

# test_dataset = EntityDataset(
#         texts=[sentence], 
#         tags=[[0] * len(sentence)]
#     )

# device = torch.device("cuda")
# pred_model = EntityModel(num_tag=num_tag)
# pred_model.load_state_dict(torch.load(MODEL_PATH))
# pred_model.to(device)

# with torch.no_grad():
#   data = test_dataset[0]
#   for k, v in data.items():
#     data[k] = v.to(device).unsqueeze(0)
#   tag, _ = pred_model(**data)

# print(
#     enc_tag.inverse_transform(
#         tag.argmax(2).cpu().numpy().reshape(-1)
#         )[:len(tokenized_sentence)]
#     )
# # print(
# #     enc_pos.inverse_transform(
# #         pos.argmax(2).cpu().numpy().reshape(-1)
# #         )[:len(tokenized_sentence)]
# #       )
device = torch.device("cuda")
# # 
# train_dataset = EntityDataset(
#         texts=train_sentences, tags=train_tag
#     )

# train_data_loader = torch.utils.data.DataLoader(
#         train_dataset, batch_size=TRAIN_BATCH_SIZE, num_workers=2
#     )

# valid_dataset = EntityDataset(
#         texts=test_sentences, tags=test_tag
#     )

# valid_data_loader = torch.utils.data.DataLoader(
#         valid_dataset, batch_size=VALID_BATCH_SIZE, num_workers=1
#     )
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# model = EntityModel(num_tag=num_tag)
# model.to(device)
# # 

for i in df["summaries"]:
  sentence = i
  tokenized_sentence = TOKENIZER.encode(sentence)
  sentence = sentence.split()
  print("******** sentence ********", sentence)
  print("******** tokenised sentence ********", tokenized_sentence)
  test_dataset = EntityDataset(
        texts=[sentence], 
        tags=[[0] * len(sentence)]
    )
  pred_model = EntityModel(num_tag=num_tag)
  pred_model.load_state_dict(torch.load(MODEL_PATH))
  pred_model.to(device)
  with torch.no_grad():
    data = test_dataset[0]
    for k, v in data.items():
      data[k] = v.to(device).unsqueeze(0)
      tag, _ = pred_model(**data)

    preds = enc_tag.inverse_transform(
        tag.argmax(2).cpu().numpy().reshape(-1)
        )[:len(tokenized_sentence)]

    for elem,cat in zip(tokenized_sentence , preds):
      print(TOKENIZER.decode([elem]), '=====>',cat)


  


  

# sentence = "seasoned backend developer. entrepreneur. open source contributor. scalable, highly-available web development: python (django), ruby (rails, sinatra), node.js, go, react js/native, angular, java. web backend scalability and performance tuning: new relic, ruby-prof, cprofile. queue-based solutions: kue, resque/sidekiq, celery, jms, rabbitmq. mobile: swift, objective-c, restkit/afnetworking, coredata, corelocation, gcd, sentestingkit, android studio, play service, retrofit. deep learning: convolutional neural network. test-driven-development: rspec, cucumber, python unittest, junit, jasmine. continuous integration/delivery: travis, jenkins, capistrano, vagrant, git, subversion, rake, maven, ant, buildout, make/gnu make. cloud/container: aws, azure, docker, docker-compose, dcos, kubernetes. devops: mesos, chef, puppet, mcollective, pxe, ipmi, nagios, zabbix. scripting: bash, python, ruby, perl. open source projects that enjoy 200+ stars on github and 100+k downloads on sourceforge., seasoned backend developer and entrepreneur.\n\nspecialties: \nscalable, highly-available web development: java, ruby (rails, sinatra), python (django), node.js.\nqueue-based solutions: resque/sidekiq, celery, jms, rabbitmq.\nweb backend performance tuning.\nios: coredata, corelocation, gcd, restkit/afnetworking, sentestingkit, swift.\ntest-driven-development: rspec, cucumber, python unittest, junit.\ncontinuous integration/delivery: jenkins, chef, capistrano, vagrant, git, subversion, rake, maven, ant, buildout, make/gnu make.\ninfrastructure-as-a-service: openstack nova, aws ec2.\nplatform-as-a-service: cloudfoundry, heroku, rightscale, enstratus, scalr, juju.\ncloud storage: hadoop hdfs, aws s3, openstack swift, mongodb.\nagile methodologies: scrum, fdd (feature-driven-development).\ndevops: chef, puppet, mcollective, pxe, ipmi, nagios, zabbix.\nscripting: bash, python, ruby, perl., programmer. entrepreneur at banian labs, pdh - networking/network engineering, pdh - network planner/provisioning, vp engineering at rhumbix"
# tokenized_sentence = TOKENIZER.encode(sentence)

# sentence = sentence.split()
# print(sentence)
# print(tokenized_sentence)

# test_dataset = EntityDataset(
#         texts=[sentence], 
#         tags=[[0] * len(sentence)]
#     )

# device = torch.device("cuda")
# pred_model = EntityModel(num_tag=num_tag)
# pred_model.load_state_dict(torch.load(MODEL_PATH))
# pred_model.to(device)

# with torch.no_grad():
#   data = test_dataset[0]
#   for k, v in data.items():
#     data[k] = v.to(device).unsqueeze(0)
#   tag, _ = pred_model(**data)

# print(
#     enc_tag.inverse_transform(
#         tag.argmax(2).cpu().numpy().reshape(-1)
#         )[:len(tokenized_sentence)]
#     )
# # print(
# #     enc_pos.inverse_transform(
# #         pos.argmax(2).cpu().numpy().reshape(-1)
# #         )[:len(tokenized_sentence)]
# #       )
preds = enc_tag.inverse_transform(
        tag.argmax(2).cpu().numpy().reshape(-1)
        )[:len(tokenized_sentence)]
TOKENIZER.decode(30000)
for elem,cat in zip(tokenized_sentence , preds):
  print(TOKENIZER.decode([elem]), '=====>',cat)
tag.argmax(2)
tokenized_sentence
enc_tag
TOKENIZER.decode(tokenized_sentence)
# if __name__ == "__main__":
#     sentences, pos, tag, enc_pos, enc_tag = process_data(config.TRAINING_FILE)
    
#     meta_data = {
#         "enc_pos": enc_pos,
#         "enc_tag": enc_tag
#     }

#     joblib.dump(meta_data, "meta.bin")

#     num_pos = len(list(enc_pos.classes_))
#     num_tag = len(list(enc_tag.classes_))

#     (
#         train_sentences,
#         test_sentences,
#         train_pos,
#         test_pos,
#         train_tag,
#         test_tag
#     ) = model_selection.train_test_split(sentences, pos, tag, random_state=42, test_size=0.1)

#     train_dataset = dataset.EntityDataset(
#         texts=train_sentences, pos=train_pos, tags=train_tag
#     )

#     train_data_loader = torch.utils.data.DataLoader(
#         train_dataset, batch_size=config.TRAIN_BATCH_SIZE, num_workers=4
#     )

#     valid_dataset = dataset.EntityDataset(
#         texts=test_sentences, pos=test_pos, tags=test_tag
#     )

#     valid_data_loader = torch.utils.data.DataLoader(
#         valid_dataset, batch_size=config.VALID_BATCH_SIZE, num_workers=1
#     )

#     device = torch.device("cuda")
#     model = EntityModel(num_tag=num_tag, num_pos=num_pos)
#     model.to(device)

#     param_optimizer = list(model.named_parameters())
#     no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
#     optimizer_parameters = [
#         {
#             "params": [
#                 p for n, p in param_optimizer if not any(nd in n for nd in no_decay)
#             ],
#             "weight_decay": 0.001,
#         },
#         {
#             "params": [
#                 p for n, p in param_optimizer if any(nd in n for nd in no_decay)
#             ],
#             "weight_decay": 0.0,
#         },
#     ]

#     num_train_steps = int(len(train_sentences) / config.TRAIN_BATCH_SIZE * config.EPOCHS)
#     optimizer = AdamW(optimizer_parameters, lr=3e-5)
#     scheduler = get_linear_schedule_with_warmup(
#         optimizer, num_warmup_steps=0, num_training_steps=num_train_steps
#     )

#     best_loss = np.inf
#     for epoch in range(config.EPOCHS):
#         train_loss = engine.train_fn(train_data_loader, model, optimizer, device, scheduler)
#         test_loss = engine.eval_fn(valid_data_loader, model, device)
#         print(f"Train Loss = {train_loss} Valid Loss = {test_loss}")
#         if test_loss < best_loss:
#             torch.save(model.state_dict(), config.MODEL_PATH)
#             best_loss = test_loss<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13384673789251045, 'fraction_numerical': 0.008677260155106026, 'mean_word_length': 3.5173934345908866, 'pattern_counts': {'":': 17, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '510031', 'license': 'MIT', 'n_tokens_mistral': 6420, 'n_tokens_neox': 6216, 'n_words': 1315, 'path': 'GPU Notebooks/BERT_Scratch/Train_NER_2.ipynb', 'repo_name': 'mohit-bags/BERT-Notebooks', 'size': '18439'}