<jupyter_start><jupyter_text>POS-Tagging__Project By__:- <NAME>- <NAME>- U<NAME>riefTagging is a kind of classification that may be defined as the automatic assignment of description to the tokens. Part-of-Speech (PoS) tagging may be defined as the process of assigning one of the parts of speech to the given word.  Problem StatementDevelop an AI model to generate the best tag for a given wore. SetupLet's install the requirements first.  Let's beginLet's see some sample data first.<jupyter_code>import os
from cfg import config
from bs4 import BeautifulSoup

train_folder_path = config['train_folder_path']
train_folders = os.listdir(train_folder_path)

if train_folders:
    random_folder = os.path.join(train_folder_path, train_folders[0])
    train_files = os.listdir(random_folder)
    if train_files:
        random_file = os.path.join(random_folder, train_files[0])
        with open(random_file) as f:
            data = f.read()
        soup = BeautifulSoup(data, 'xml')
        count = 0
        for tag in soup.find_all('w'):
            count += 1
            print(tag)
            if count > 10:
                break<jupyter_output><w c5="AJ0" hw="crude" pos="ADJ">Crude </w>
<w c5="NN1" hw="justice" pos="SUBST">justice</w>
<w c5="VVB-NN1" hw="deal" pos="VERB">Deal </w>
<w c5="PRP" hw="with" pos="PREP">with </w>
<w c5="NN1" hw="oil" pos="SUBST">oil </w>
<w c5="NN2" hw="spill" pos="SUBST">spills </w>
<w c5="PRP-AVP" hw="in" pos="PREP">in </w>
<w c5="AT0" hw="a" pos="ART">a </w>
<w c5="NN1" hw="way" pos="SUBST">way </w>
<w c5="CJT" hw="that" pos="CONJ">that </w>
<w c5="VVZ" hw="clean" pos="VERB">cleans </w><jupyter_text>__We need to define some important variables:__- lowercase: Convert all data to lowercase before processing<jupyter_code>lowercase = False
output_folder_path = config['output_folder_path']

if not (os.path.exists(output_folder_path) and os.path.isdir(output_folder_path)):
    os.mkdir(output_folder_path)<jupyter_output><empty_output><jupyter_text>We need to process and clean the train data before proceeding further.<jupyter_code>import corpus_processor
from library import dirWalk

input_folder_path = config['train_folder_path']

file_list = []

output_file_path = os.path.join(output_folder_path,'train-corpus_preprocessed.txt')
file_list = dirWalk(input_folder_path, file_list)

corpus_processor.process(file_list, output_file_path, lowercase)

print('Preprocessing complete')<jupyter_output>Preprocessing complete<jupyter_text>Let's see some of the preprocessed data<jupyter_code>output_file = os.path.join(output_folder_path, 'train-corpus_preprocessed.txt')
with open(output_file) as f:
    for i in range(10):
        data = f.readline().strip()
        print(data)<jupyter_output>Crude_AJ0
justice_NN1
Deal_VVB-NN1
with_PRP
oil_NN1
spills_NN2
in_PRP-AVP
a_AT0
way_NN1
that_CJT<jupyter_text>Each of these word can occur hundred of times. Counting the frequency of the word tag combination can speed the process.<jupyter_code>from library import clean

input_file_path = os.path.join(output_folder_path, 'train-corpus_preprocessed.txt')
with open(input_file_path, 'r') as f:
    preprocessed_list = f.readlines()
trimmed_list = [entry.strip('\n') for entry in preprocessed_list]
del preprocessed_list
train_dictionary = dict()
for entry in trimmed_list:
    if entry not in train_dictionary:
        train_dictionary[entry] = 1
    else:
        train_dictionary[entry] += 1
del trimmed_list
train_dictionary = clean(train_dictionary)

print('Counting word-tag combinations complete')<jupyter_output>Counting word-tag combinations complete<jupyter_text>Let's see some of the dictionary entries.<jupyter_code>count = 0
for key, value in train_dictionary.items():
    count += 1
    if count > 10:
        break
    print(key, ':', value)<jupyter_output>Crude_AJ0 : 7.5
justice_NN1 : 381
with_PRP : 61930
oil_NN1 : 915.0
spills_NN2 : 14.0
a_AT0 : 198465
way_NN1 : 8761
that_CJT : 65745.5
cleans_VVZ : 6.5
up_AVP : 17454.5<jupyter_text>To understand the distribution of tags and words, let us find out the top 10 words and top 10 tags.<jupyter_code>import itertools


words = dict()
tags = dict()

for key, value in train_dictionary.items():
    word, tag = key.split('_')

    if word not in words:
        words[word] = value
    else:
        words[word] = words[word] + value

    if tag not in tags:
        tags[tag] = value
    else:
        tags[tag] = tags[tag] + value
        
sorted_words = {key: value for key, value in sorted(
    words.items(), key=lambda item: item[1], reverse=True)}
sorted_tags = {key: value for key, value in sorted(
    tags.items(), key=lambda item: item[1], reverse=True)}

top_words_list = [key for key in itertools.islice(sorted_words.keys(), 10)]
top_tags_list = [key for key in itertools.islice(sorted_tags.keys(), 10)]<jupyter_output><empty_output><jupyter_text>The top 10 tags are:<jupyter_code>for tag in top_tags_list:
    print(tag)<jupyter_output>NN1
PUN
AT0
PRP
AJ0
NN2
NP0
AV0
PNP
CJC<jupyter_text>The top 10 words are:<jupyter_code>for tag in top_words_list:
    print(tag)<jupyter_output>the
,
.
of
to
and
a
in
is
that<jupyter_text>Let us plot some graphs to further understand the dataset.__Plotting Tags vs Frequency Bar Graph__<jupyter_code>import matplotlib.pyplot as plt

plt.title('Tags vs Frequency')
plt.ylabel('Frequency')
plt.xlabel('Tags')
plt.bar(range(0, len(sorted_tags) * 2, 2), list(sorted_tags.values()),
        align='center', width=1)
plt.xticks(range(0, len(sorted_tags) * 2, 2),
           list(sorted_tags.keys()), rotation=90, fontsize=8)
fig_size = plt.gcf().get_size_inches()
plt.gcf().set_size_inches(fig_size[0] * 2, fig_size[1] * 1, forward=True)
plt.tight_layout(pad=0.5)
ax = plt.gca()
ax.set_xlim(left=-2)
plt.show()
plt.close()<jupyter_output><empty_output><jupyter_text>__Plotting Tags Pie Chart__<jupyter_code>import numpy as np

labels = list(itertools.islice(sorted_tags.keys(), 10))
labels.append('Other')
sizes = list(itertools.islice(sorted_tags.values(), 10))
total = sum(sorted_tags.values())
other_size = total - sum(itertools.islice(sorted_tags.values(), 10))
sizes.append(other_size)

cmap = plt.get_cmap("tab20c")
colors = cmap(np.arange(11))
plt.pie(sizes, labels=labels, colors=colors,
        autopct='%1.1f%%', shadow=True, startangle=140)

plt.axis('equal')
plt.show()
plt.close()<jupyter_output><empty_output><jupyter_text>__Plotting Words Pie Chart__<jupyter_code>plt.title('Top 10 words')
labels = list(itertools.islice(sorted_words.keys(), 10))
labels.append('Other')
sizes = list(itertools.islice(sorted_words.values(), 10))
total = sum(sorted_words.values())
other_size = total - sum(itertools.islice(sorted_words.values(), 10))
sizes.append(other_size)
explode = (0, 0, 0, 0.1, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0)
cmap = plt.get_cmap("tab20c")
colors = cmap(np.arange(11) * 2)
plt.pie(sizes, labels=labels, colors=colors, explode=explode,
        autopct='%1.1f%%', shadow=True, startangle=140)

plt.axis('equal')
plt.show()
plt.close()<jupyter_output><empty_output><jupyter_text>Cleanup resources (Don't cleanup unless you don't want to draw above graphs again)<jupyter_code>del sorted_words<jupyter_output><empty_output><jupyter_text>Let us process the Test data also.<jupyter_code>input_folder_path = config['test_folder_path']

file_list = []

output_file_path = os.path.join(output_folder_path,'test-corpus_preprocessed.txt')
file_list = dirWalk(input_folder_path, file_list)

corpus_processor.process(file_list, output_file_path, lowercase)
output_file_path = os.path.join(output_folder_path,'test-corpus_preprocessed.txt')

with open(output_file_path, 'r') as f:
    preprocessed_list = f.readlines()
trimmed_list = [entry.strip('\n') for entry in preprocessed_list]
del preprocessed_list
test_dictionary = dict()
for entry in trimmed_list:
    if entry not in test_dictionary:
        test_dictionary[entry] = 1
    else:
        test_dictionary[entry] += 1
del trimmed_list
test_dictionary = clean(test_dictionary)

print('Test data processed.')<jupyter_output>Test data processed.<jupyter_text>Naive Probability Model Train the model<jupyter_code>frequency_map = dict()

for key, value in train_dictionary.items():
    word, tag = key.split('_')

    if word not in frequency_map:
        frequency_map[word] = dict()
        frequency_map[word][tag] = value
    else:
        if tag not in frequency_map[word]:
            frequency_map[word][tag] = value
        else:
            frequency_map[word][tag] += value

for tag_dictionary in frequency_map.values():
    total = sum(tag_dictionary.values())
    for tag in tag_dictionary.keys():
        tag_dictionary[tag] = round(
            tag_dictionary[tag] / total, 7)

tagger_map = dict()
for word, tag_dictionary in frequency_map.items():
    tagger_map[word] = max(tag_dictionary, key=tag_dictionary.get)<jupyter_output><empty_output><jupyter_text>Test the model<jupyter_code>tagged_words = dict()
best_tag = top_tags_list[0]

for key, value in test_dictionary.items():
    word, expected_tag = key.split('_')

    if key not in tagged_words:
        if word in tagger_map:
            assigned_tag = tagger_map[word]
        elif word.lower() in tagger_map:
            assigned_tag = tagger_map[word.lower()]
        elif word.upper() in tagger_map:
            assigned_tag = tagger_map[word.upper()]
        elif word.capitalize() in tagger_map:
            assigned_tag = tagger_map[word.capitalize()]
        else:
            assigned_tag = best_tag
        tag_dictionary = {
            "word": word,
            "frequency": value,
            "expected_tag": expected_tag,
            "assigned_tag": assigned_tag
        }
        tagged_words[key] = tag_dictionary
    else:
        tagged_words[key]["frequency"] = tagged_words[key]["frequency"] + value<jupyter_output><empty_output><jupyter_text>Analysing the modelGenerating confusion matrix<jupyter_code>import pandas as pd

confusion_matrix = dict.fromkeys(sorted_tags.keys())
for key in confusion_matrix.keys():
    confusion_matrix[key] = dict.fromkeys(sorted_tags.keys(), 0)

for value in tagged_words.values():
    actual_tag = value['expected_tag']
    predicted_tag = value['assigned_tag']
    frequency = value['frequency']
    if predicted_tag in confusion_matrix:
        confusion_matrix[actual_tag][predicted_tag] += frequency

dataframe = pd.DataFrame(confusion_matrix)
dataframe = dataframe.style.background_gradient()<jupyter_output><empty_output><jupyter_text>Let's see the dataframe<jupyter_code>dataframe<jupyter_output><empty_output><jupyter_text>Calculating the accuracy of the model<jupyter_code>from library import judge

average_f1_score, weighted_f1_score = judge(confusion_matrix)
print("Average F1 score:", average_f1_score)
print("Weighted F1 score:", weighted_f1_score)<jupyter_output>Average F1 score: 0.851826028759796
Weighted F1 score: 0.9048713765307177
{'fraction_non_alphanumeric': 0.11800385993934381, 'fraction_numerical': 0.02031063321385902, 'mean_word_length': 4.84739387426115, 'pattern_counts': {'":': 4, '<': 93, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '399813', 'license': 'MIT', 'n_tokens_mistral': 4024, 'n_tokens_neox': 3799, 'n_words': 867, 'path': 'naive.ipynb', 'repo_name': 'wistic/POS-Tagging-jupyter', 'size': '10908'}