<jupyter_start><jupyter_text>TEXT data cleaner and tokenizerwe use NLTK, Regex, BeautifulSoup, inflect to clean the parsed text data (json file), tokenize them into sentences and restore them as JSON file.Resources used to create this code are as following:*   Text processing *  Tokenization --------------------------- Step 1: Importing all the neccesary libraries<jupyter_code>import pandas as pd
import re, string, unicodedata
import inflect
from collections import defaultdict
from bs4 import BeautifulSoup as BS
import spacy
from nltk.corpus import stopwords
STOPWORDS = set(stopwords.words('english'))
import json
import nltk.data
from nltk.tokenize import sent_tokenize 
from nltk.stem import LancasterStemmer, WordNetLemmatizer
import numpy as np<jupyter_output><empty_output><jupyter_text>Step 2: Openning, premilinary cleaning, and tokenizing the text data into sentences<jupyter_code>def open_clean_tokenize(filepath):
    """ filepath is the path to the json file to be openned. This function opens the file, do some basic cleaning (makes text lowercase, remoeves  and removes unwanted symboles and tokenize the data in to sentences
"""
    with open(filepath) as json_file:
        corpus = json.load(json_file)
        #corpus = pd.read_json(json_file, orient ='index') 
    corpus = str(corpus)
    corpus = BS(corpus, 'lxml').text #HTML decoding. BeautifulSoup's text attribute will return a string stripped of any HTML tags and metadata.
    corpus = corpus.lower()
    corpus = re.sub(r'\w*\d\w*', '', corpus)
    corpus = re.sub(r'http\S+', '', corpus) # removes url
    corpus = re.compile('[/(){}\[\]\|@;#:]').sub('', corpus) #replace REPLACE_BY_SPACE_RE symbols by space in text. substitute the matched string in REPLACE_BY_SPACE_RE with space.
    corpus = re.compile('(\$[a-zA-Z_][0-9a-zA-Z_]*)').sub('', corpus) #remove symbols from text.
    clean_tokenize = nltk.tokenize.sent_tokenize(corpus) # tokenizes data into sentences
    
    return clean_tokenize<jupyter_output><empty_output><jupyter_text>Step 3: Further cleaning (remove punctuations, numbers, and stop words) the tokenized data in setp 2 for to use in Word2vec<jupyter_code>def remove_punctuation(corpus):
    """Remove punctuation from list of tokenized corpus"""
    new_corpus = []
    for token in corpus:
        new_token = re.sub(r'[^\w\s]', '', token)
        if new_token != '':
            new_corpus.append(new_token)
    return new_corpus

def replace_numbers(corpus):
    """Replace all interger occurrences in list of tokenized words with textual representation"""
    p = inflect.engine()
    new_corpus = []
    for token in corpus:
        if token.isdigit():
            new_token = p.number_to_words(token)
            new_corpus.append(new_token)
        else:
            new_corpus.append(token)
    return new_corpus

def remove_stopwords(corpus):
    """Remove stop words from list of tokenized words"""
    new_corpus = []
    for token in corpus:
        if token not in stopwords.words('english'):
            new_corpus.append(token)
    return new_corpus

def normalize(corpus):
    corpus = remove_punctuation(corpus)
    corpus = replace_numbers(corpus)
    corpus = remove_stopwords(corpus)
    return corpus<jupyter_output><empty_output><jupyter_text>Step 4: Let's run the above two functions as below<jupyter_code>filepath = 'Basic Principles of Organic Chemistry_Roberts and Caserio'
corpus = open_clean_tokenize(filepath)
tokens = normalize(corpus)<jupyter_output><empty_output><jupyter_text>Stemming and lemmatizationWe can further process the cleanned and tokenized data as follows:<jupyter_code>def stem_words(tokens):
    """Stem words in list of tokenized words"""
    tokens = normalize(corpus)
    stemmer = LancasterStemmer()
    stems = []
    for token in tokens:
        stem = stemmer.stem(token)
        stems.append(stem)
    return stems

def lemmatize_verbs(tokens):
    """Lemmatize verbs in list of tokenized words"""
    lemmatizer = WordNetLemmatizer()
    lemmas = []
    for token in tokens:
        lemma = lemmatizer.lemmatize(token, pos='v')
        lemmas.append(lemma)
    return lemmas
def stem_and_lemmatize(tokens):
    stems = stem_words(tokens)
    lemmas = lemmatize_verbs(tokens)
    return stems, lemmas
final_data = stem_and_lemmatize(tokens)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0978085351787774, 'fraction_numerical': 0.0020761245674740486, 'mean_word_length': 4.03016241299304, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '448355', 'license': 'MIT', 'n_tokens_mistral': 1402, 'n_tokens_neox': 1340, 'n_words': 402, 'path': 'Rukiya/text_data_cleaner.ipynb', 'repo_name': 'JonathanOnorato/ChemLP', 'size': '4335'}