<jupyter_start><jupyter_text>Loading Data<jupyter_code>%reload_ext autoreload
%autoreload 2
import os, json, random, re
import pandas as pd
from tqdm import tqdm
import utils
data_f = utils.data_f
train_csv = pd.read_csv(data_f + 'train.csv')

print(train_csv.shape, 'train_csv shape')
train_csv.sample(10)
# documents
print(train_csv.Id.unique().shape, 'unique ids') # repeated IDs means documents have multiple dataset labels
most_repeated_ids = train_csv.Id.value_counts()[:10]
print(most_repeated_ids)
## datasets
print(train_csv.cleaned_label.unique().shape, 'unique_cleaned labels (# of datasets)')
print(train_csv.cleaned_label.value_counts()[:10], 'most reffered datasets, with freq')
train_csv.cleaned_label.value_counts().describe() # most of the top dataset are mentioned over 10 times, while each dataset is mentioned at least twice
import prepare_data as prep
ids = train_csv.Id.sample(10).unique()
labels_map = {i: train_csv[train_csv.Id==i]['cleaned_label'].values.tolist() for i in ids}

data = prep.DataPrep(labels_map)
tagged_data = data.prep()
tagged_data[0]
# one example
d = data.meta['05e7e2b0-22a6-44f4-8181-c4572d954fa3_1']
text = d['clean_text']
labels = data.labels_map['05e7e2b0-22a6-44f4-8181-c4572d954fa3']
idx = prep.get_indx(text, labels)
tokens, tags = prep.split_and_tag(text, idx)
print([(i, j) for i, j in zip(tokens, tags) if j!='O'])
# more examples
for c, i in enumerate(tagged_data[:10]):
    tokens = i['tokens']    
    tags = i['tags']
    para_id = i['id']
    doc_id = para_id.split('_')[0]
    labels = data.labels_map[doc_id]
    text = data.meta[para_id]['clean_text']
    
    tagging = ' '.join([str((i, j)) for i, j in zip(tokens, tags) if j != 'O'])
    
    if tagging != []:
        print(para_id)
        print('lables=', labels, '\n',
            'tagging=', tagging)
utils.pkl_save(data, utils.data_f+'save/prep_data.pkl')
utils.pkl_save(tagged_data, utils.data_f+'save/tagged_data.pkl')
import pipeline as pipe


tagged_data = utils.read_pkl(utils.data_f+'save/tagged_data.pkl')
mapping = dict(
    id=[i['id'] for i in tagged_data],
    tokens = [i['tokens'] for i in tagged_data],
    ner_tags = [i['tags'] for i in tagged_data]
)
data = pipe.Dataset.from_dict(mapping=mapping)

data['tokens'][3][:10]
s = pipe.tokenize_and_align(data[:3])
# s['labels'][2]
dataset = pipe.read_prepared_data(utils.data_f+'save/tagged_data.pkl')
pipe.tokenizer??
r_vocab = {j: i for i, j in pipe.tokenizer.vocab.items()}

for i in [101, 7592, 2433, 1996, 2060, 2217, 20066, 10735, 10288, 8787, 102]:
    print(r_vocab[i])
pipe.tokenizer('hello form the other side sapiosexual')
s = dataset[23]
s1, s2, s3 = s['input_ids'], s['labels'], s['attention_mask']

print(list(map(len, [s1, s2, s3])))

s4 = [i if j!=0 and j!=-100 else 0 for i, j in zip(s1, s2)]
s5 = [i if j==1 else 0 for i, j in zip(s1, s3)]

pipe.tokenizer.decode(s5)
labels_map['c56f58c4-45f3-4a32-bea8-4f1f3c9799cd']
import transformers
transformers.__version__

task = "ner" # Should be one of "ner", "pos" or "chunk"
model_checkpoint = "distilbert-base-uncased"
batch_size = 16

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)


# a = tokenizer("Hello, this is one sentence!")



# s = sections_target[4]
# text_to_tokenize = s.text
# target_char_index = s.text.index(s.target)
# target_span = (target_char_index, target_char_index + len(s.target))
def get_dataset_dict(sections):
    tagged_data = [tag_section(s) for s in sections]
    return dict(
        mapping = dict(
            id=list(range(len(tagged_data))),
            tokens=[i for i,j in tagged_data],
            ner_tags=[list(map(ner_tags.get, j)) for i,j in tagged_data]            
        )
    )

from datasets import Dataset, load_metric

data = Dataset.from_dict(**get_dataset_dict(sections_target))

def tokenize_and_align(examples):
    tokenized_inputs = tokenizer(examples["tokens"], padding='max_length', truncation=True, is_split_into_words=True)

    labels = []
    for i, label in enumerate(examples[f"ner_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:
            if word_idx is None:
                label_ids.append(-100)
            # We set the label for the first token of each word.
            elif word_idx != previous_word_idx:
                label_ids.append(label[word_idx])
            # For the other tokens in a word, we set the label to either the current label or -100, depending on
            # the label_all_tokens flag.
            else:
                label_ids.append(label[word_idx])
            previous_word_idx = word_idx

        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    return tokenized_inputs
tokenize_and_align(data[:2])
tokenized_data = data.map(tokenize_and_align, batched=True)
label_list = list(ner_tags.keys())
from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint, num_labels=len(label_list))
args = TrainingArguments(
    f"test-ner",
    evaluation_strategy = "epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    num_train_epochs=3,
    weight_decay=0.01,
)
from transformers import DataCollatorForTokenClassification
data_collator = DataCollatorForTokenClassification(tokenizer)
# !pip install seqeval
metric = load_metric("seqeval")
labels = data[f"{task}_tags"]
metric.compute(predictions=labels, references=labels)
import numpy as np

def compute_metrics(p):
    predictions, labels = p
    predictions = np.argmax(predictions, axis=2)

    # Remove ignored index (special tokens)
    true_predictions = [
        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]
    true_labels = [
        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]

    results = metric.compute(predictions=true_predictions, references=true_labels)
    return {
        "precision": results["overall_precision"],
        "recall": results["overall_recall"],
        "f1": results["overall_f1"],
        "accuracy": results["overall_accuracy"],
    }
trainer = Trainer(
    model,
    args,
    train_dataset=tokenized_data,
    eval_dataset=tokenized_data,
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)
trainer.train()
batch_size=4
!ls
test_csv = pd.read_csv(data_f + 'sample_submission.csv')
test_csv
def prepare_datasets(csv):
    sections = [Section(jdata, csv_ind) for csv_ind in tqdm(csv.index) for jdata in get_json(from_csv(csv_ind, 'Id', csv))]
    data = Dataset.from_dict(**get_dataset_dict(sections_target))
    tokenized_data = data.map(tokenize_and_align, batched=True)
    return tokenized_data, sections   
    
test_tokenized, test_sections = prepare_datasets(test_csv)
test_tokenized
test_tokenized.remove_columns(['labels', 'ner_tags'])
model()
trainer.compute_metrics = None
predictions, labels, _ = trainer.predict(test_tokenized)
predictions = np.argmax(predictions, axis=2)

# Remove ignored index (special tokens)
# true_predictions = [
#     [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
#     for prediction, label in zip(predictions, labels)
# ]
# true_labels = [
#     [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
#     for prediction, label in zip(predictions, labels)
# ]

# results = metric.compute(predictions=true_predictions, references=true_labels)
# results
# predictions = numpy.array([[label_list[p] for p in pred] for pred in predictions])
def decode_labels(data, lables, skip='[SEP]'):
    
    for ix in tqdm(range(len(data))):
        # ix = 4
        predicted = [(i, j) for i, j in enumerate(labels[ix]) if j in [1, 2]]
        s = [(data[ix]['input_ids'][i], j) for i, j in predicted]
        s = tokenizer.decode([i for i, j in s])
        if s not in skip: print(ix, s)
len(test_tokenized)
# a page of requirement of KG paper
random_ids = random.choices(range(0, 700), k=5)
data = [{'input_ids': tokenized_data['input_ids'][i]} for i in random_ids]
labels = [tokenized_data['labels'][i] for i in random_ids]

decode_labels(data, labels)
random_ids
train_csv.iloc[random_ids]
type(model)
trainer.save_model??
# !mkdir -p saved_models/coleridge/distill-bert-finetuned1
trainer.save_model('saved_models/coleridge/distill-bert-finetuned1')
!du -sh saved_models/
!free
!free -h<jupyter_output>total        used        free      shared  buff/cache   available
Mem:            12G        4.2G        3.5G         24M        4.9G        8.1G
Swap:            0B          0B          0B
{'fraction_non_alphanumeric': 0.1238988028009939, 'fraction_numerical': 0.023831036819516604, 'mean_word_length': 4.0861573808156235, 'pattern_counts': {'":': 4, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '99053', 'license': 'Apache-2.0', 'n_tokens_mistral': 3220, 'n_tokens_neox': 3082, 'n_words': 720, 'path': 'notebook.ipynb', 'repo_name': 'violetcodes/hiking-ridge', 'size': '8854'}