<jupyter_start><jupyter_text>The **Sentence**, **Word** and **POS**([Part of Speech](https://pythonprogramming.net/part-of-speech-tagging-nltk-tutorial/)) columns will represent the **feature X**, and the **Tag** column will represent right **label Y**<jupyter_code>def get_dict_map(data, token_or_tag):
    tok2idx = {}
    idx2tok = {}
    
    if token_or_tag == 'token':
        vocab = list(set(data['Word'].to_list()))
    elif token_or_tag == 'tag':
        vocab = list(set(data['Tag'].to_list()))
    
    idx2tok = {idx:tok for  idx, tok in enumerate(vocab)}
    tok2idx = {tok:idx for  idx, tok in enumerate(vocab)}
    return tok2idx, idx2tok

token2idx, idx2token = get_dict_map(data, 'token')
tag2idx, idx2tag = get_dict_map(data, 'tag')

data['Word_idx'] = data['Word'].map(token2idx)
data['Tag_idx'] = data['Tag'].map(tag2idx)

#Replace the NaN value in Sentence# with related Sentence number
data_fillna = data.fillna(method='ffill', axis=0)

data_fillna

# Groupby and collect columns
data_group = data_fillna.groupby(['Sentence #'],as_index=False)['Word', 'POS', 'Tag', 'Word_idx', 'Tag_idx'].agg(lambda x: list(x))

data_group['String'] = data_group['Sentence #'].apply(lambda x: x[:10])
data_group['Integer'] = data_group['Sentence #'].apply(lambda x: int(x[10:]))
data_group = data_group.sort_values(by=['String', 'Integer'])
data_group
import nltk
from nltk.tokenize import word_tokenize
from nltk.tag import pos_tag
ex = 'European authorities fined Google a record $5.1 billion on Wednesday for abusing its power in the mobile phone market and ordered the company to alter its practices'
def preprocess(sent):
    sent = nltk.word_tokenize(sent)
    sent = nltk.pos_tag(sent)
    return sent
sent = preprocess(ex)
sent<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1489841986455982, 'fraction_numerical': 0.01072234762979684, 'mean_word_length': 5.377697841726619, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '97119', 'license': 'MIT', 'n_tokens_mistral': 629, 'n_tokens_neox': 617, 'n_words': 155, 'path': 'Name Entity Recognition/Name Entity Recognition.ipynb', 'repo_name': 'Zheng-Lu/NLP-Projects', 'size': '1772'}