<jupyter_start><jupyter_text>download libraries<jupyter_code>!pip install emoji --upgrade
import nltk
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('sentiwordnet')
nltk.download('averaged_perceptron_tagger')<jupyter_output>Collecting emoji
[?25l  Downloading https://files.pythonhosted.org/packages/40/8d/521be7f0091fe0f2ae690cc044faf43e3445e0ff33c574eae752dd7e39fa/emoji-0.5.4.tar.gz (43kB)
[K     |████████████████████████████████| 51kB 1.7MB/s 
[?25hBuilding wheels for collected packages: emoji
  Building wheel for emoji (setup.py) ... [?25l[?25hdone
  Created wheel for emoji: filename=emoji-0.5.4-cp36-none-any.whl size=42176 sha256=2993ee6c88699e92d18115300a4e601621e78559ade4ba2bf6b0574afbcfcedf
  Stored in directory: /root/.cache/pip/wheels/2a/a9/0a/4f8e8cce8074232aba240caca3fade315bb49fac68808d1a9c
Successfully built emoji
Installing collected packages: emoji
Successfully installed emoji-0.5.4
[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.
[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data]   Unzipping corpor[...]<jupyter_text>Import necessary libraries for data cleaning, sentiment analysis and word embedding<jupyter_code>import re # for regular expressions
import json
import emoji

import pickle
from datetime import datetime

import numpy as np
import pandas as pd
pd.set_option("display.max_colwidth", 200)

from nltk import pos_tag, sent_tokenize, word_tokenize
from nltk.corpus import stopwords, wordnet as wn, sentiwordnet as swn
from nltk.stem import WordNetLemmatizer

import matplotlib.pyplot as plt 
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")

%matplotlib inline
%cd "/content/drive/My Drive/dataScience/projects/twitterSentimentAnalysis/dataset"
# !pwd<jupyter_output>/content/drive/My Drive/dataScience/projects/twitterSentimentAnalysis/dataset<jupyter_text>Define dictionary for contractions of words<jupyter_code># Dictionary of contarctions. source for contractions: https://en.wikipedia.org/wiki/Contraction_%28grammar%29
contractions_dict = {
        "ain't":"is not",
        "amn't":"am not",
        "aren't":"are not",
        "can't":"cannot",
        "'cause":"because",
        "couldn't":"could not",
        "couldn't've":"could not have",
        "could've":"could have",
        "daren't":"dare not",
        "daresn't":"dare not",
        "dasn't":"dare not",
        "didn't":"did not",
        "doesn't":"does not",
        "don't":"do not",
        "e'er":"ever",
        "em":"them",
        "everyone's":"everyone is",
        "finna":"fixing to",
        "gimme":"give me",
        "gonna":"going to",
        "gon't":"go not",
        "gotta":"got to",
        "hadn't":"had not",
        "hasn't":"has not",
        "haven't":"have not",
        "he'd":"he would",
        "he'll":"he will",
        "he's":"he is",
        "he've":"he have",
        "how'd":"how would",
        "how'll":"how will",
        "how're":"how are",
        "how's":"how is",
        "I'd":"I would",
        "I'll":"I will",
        "I'm":"I am",
        "I'm'a":"I am about to",
        "I'm'o":"I am going to",
        "isn't":"is not",
        "it'd":"it would",
        "it'll":"it will",
        "it's":"it is",
        "I've":"I have",
        "kinda":"kind of",
        "let's":"let us",
        "mayn't":"may not",
        "may've":"may have",
        "mightn't":"might not",
        "might've":"might have",
        "mustn't":"must not",
        "mustn't've":"must not have",
        "must've":"must have",
        "needn't":"need not",
        "ne'er":"never",
        "o'":"of",
        "o'er":"over",
        "ol'":"old",
        "oughtn't":"ought not",
        "shalln't":"shall not",
        "shan't":"shall not",
        "she'd":"she would",
        "she'll":"she will",
        "she's":"she is",
        "shouldn't":"should not",
        "shouldn't've":"should not have",
        "should've":"should have",
        "somebody's":"somebody is",
        "someone's":"someone is",
        "something's":"something is",
        "that'd":"that would",
        "that'll":"that will",
        "that're":"that are",
        "that's":"that is",
        "there'd":"there would",
        "there'll":"there will",
        "there're":"there are",
        "there's":"there is",
        "these're":"these are",
        "they'd":"they would",
        "they'll":"they will",
        "they're":"they are",
        "they've":"they have",
        "this's":"this is",
        "those're":"those are",
        "'tis":"it is",
        "'twas":"it was",
        "wanna":"want to",
        "wasn't":"was not",
        "we'd":"we would",
        "we'd've":"we would have",
        "we'll":"we will",
        "we're":"we are",
        "weren't":"were not",
        "we've":"we have",
        "what'd":"what did",
        "what'll":"what will",
        "what're":"what are",
        "what's":"what is",
        "what've":"what have",
        "when's":"when is",
        "where'd":"where did",
        "where're":"where are",
        "where's":"where is",
        "where've":"where have",
        "which's":"which is",
        "who'd":"who would",
        "who'd've":"who would have",
        "who'll":"who will",
        "who're":"who are",
        "who's":"who is",
        "who've":"who have",
        "why'd":"why did",
        "why're":"why are",
        "why's":"why is",
        "won't":"will not",
        "wouldn't":"would not",
        "would've":"would have",
        "y'all":"you all",
        "you'd":"you would",
        "you'll":"you will",
        "you're":"you are",
        "you've":"you have"
        }<jupyter_output><empty_output><jupyter_text>Read json formatted raw data as a DataFrame> Indented block<jupyter_code># Input data files are available in the "dataset/" directory.
# Read the input raw data and parse only the necessary columns from the json file.

def populate_tweet_df(tweets):
    """
    This function takes tweets list as argument and returns a dataframe of the tweets.
    """
    df = pd.DataFrame()
    df['date'] = [datetime.strptime(tweet['created_at'],'%a %b %d %H:%M:%S %z %Y') for tweet in tweets if (tweet['lang'] == 'en')]
    df['text'] = [tweet['text'] for tweet in tweets if (tweet['lang'] == 'en')]
    # df['text'] = list(map(lambda tweet:tweet['text'], filter(lambda tweet: tweet['lang']=='en', tweets)))
    return df

if __name__ == '__main__':
    tweet_file1 = ("tweets_parisagreement_09-03-2020.txt")
    tweets = []
    with open(tweet_file1, 'r') as file:
        for line in file.readlines():
            tweets.append(json.loads(line))
    
    tweet_file2 = ("tweets_parisagreement_20-04-2020.txt")
    with open(tweet_file2, 'r') as file:
        for line in file.readlines():
            tweets.append(json.loads(line))

    tweets_df = populate_tweet_df(tweets)
    print("Executed Successfully")

tweets_df.head()<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code># Setting stopwords
all_stopwords = set(stopwords.words('english'))
exclude_stopwords = ['no', 'not']
for sw in exclude_stopwords:
    all_stopwords.remove(sw)

# Initializing Lemmatizer
lemmatizer = WordNetLemmatizer()

def toLowerCase(text):
    '''
    Returns the text in lowercase.
    '''
    return text.lower()


def removeUrls(text):
    # return re.sub('((www\.[^\s]+)|(https?://[^\s]+))','', text)
    return re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b','', text)

def removeSpecialChar(text):
    '''
    Removes special characters which are generally found in tweets.
    '''
    # Convert @username to empty strings
    # text = re.sub('@[^\s]+', '', text)
    text = re.sub('@\S+','',text)

    # Remove 'RT' from retweets
    text = re.sub(r'\brt\b', '', text)

    # Remove additional white spaces
    text = re.sub('[\s]+', ' ', text)

    # Replace #hastags with white space
    # text = re.sub(r'#([^\s]+)', '', text)
    text = re.sub(r"#\S+", "", text)

    # Trims the tweet
    text = text.strip('\'"')

    return text


def removeNonAlpha(text):
    '''
    Remove all characters which are not alphabets, numbers or whitespaces.
    '''
    return re.sub('[^A-Za-z0-9 ]+', '', text)


def removeStopWords(text):
    # Remove stop words using NLTK's list of stop words in the corpus module. 
    tokens_without_sw = [word for word in word_tokenize(text) if not word in all_stopwords]

    return (tokens_without_sw)


def handleEmojis(text):
    '''
    This function handles sentiments expressed through emoticons by using Python's 'emoji package'.
    Replace all emoticons with the expression they represent in plain English.
    '''
    text = emoji.demojize(text)
    text = text.replace(":"," ")
    text = text.replace("_"," ")
    text = ' '.join(text.split())
    return text


def handleContractions(text):
    '''
    expand shortened words, e.g. convert "don't" to "do not"
    '''
    contracted_tokens = [contractions_dict[word] if word in contractions_dict else word for word in word_tokenize(text)]
    return " ".join(contracted_tokens)


def get_wordnet_pos(word):
    '''
    Map POS tag to the format wordnet lemmatizer would accept. Returning 'NOUN' as default tag.
    ADJ: adjective,    ADP: adposition,    ADV: adverb,    CONJ: conjunction	
    DET: determiner,article,    NOUN: noun,    NUM: numeral,    PRT: particle
    PRON: pronoun,    VERB: verb,    .: punctuation marks	. , ; !,    X: other	
    '''
    tag = pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wn.ADJ,
                "N": wn.NOUN,
                "V": wn.VERB,
                "R": wn.ADV}

    return tag_dict.get(tag, wn.NOUN)


def wordLemmatization(text):
    '''
    Lemmatizing the text with the appropriate POS tag
    result: ['he', 'kept', 'eat', 'while', 'we', 'be', 'talk']
    '''
    lemma_tokens = [lemmatizer.lemmatize(word, get_wordnet_pos(word)) for word in word_tokenize(text)]
    return " ".join(lemma_tokens)

def processTweets(tweet):
    '''
    Process the raw tweets and convert them into usable data for sentiment analysis. 
    '''
    # Convert to lower case
    tweet = toLowerCase(tweet)

    # Remove all Special characters
    tweet = removeSpecialChar(tweet)

    # Remove urls
    tweet = removeUrls(tweet)
    
    # Replacing contractions with full words
    tweet = handleContractions(tweet)
    
    # Replace emoticons with word expressions
    tweet = handleEmojis(tweet)
    
    # # Remove all Special characters
    tweet = removeSpecialChar(tweet)
    
    # Remove non alpha characters
    tweet = removeNonAlpha(tweet)
    
    # converting a word to its base form
    tweet = wordLemmatization(tweet)
    
    # Remove stop words
    tweet = removeStopWords(tweet)

    return tweet

if __name__ == '__main__':
    tweets = tweets_df['text']
    processed_tweets = []
    processed_df = pd.DataFrame(columns=['date','text','clean_text'])
    for tweet in tweets:
        processed_tweets.append(processTweets(tweet))
    processed_df['date'] = tweets_df['date']
    processed_df['text'] = tweets_df['text']
    processed_df['clean_text'] = processed_tweets
    print("Executed Successfully")
tweet1 = '''RT @AngholichiGoli Ghana will share their progress on cooperative approaches aligned with #ParisAgreemet'''
tweet2 = '''RT @frances_cutler: are they massively trying to massage the figures to hide the fact😡❓'''
print(processTweets(tweet1))
print(processTweets(tweet2))
processed_df.head()<jupyter_output><empty_output><jupyter_text>Save processed dataframe for easy reusability<jupyter_code>cleaned_df = open("cleaned_df.pickle","wb")
pickle.dump(processed_df, cleaned_df)
cleaned_df = open("cleaned_df.pickle", "rb")
processed_df = pickle.load(cleaned_df)<jupyter_output><empty_output><jupyter_text>Hypernyms<jupyter_code>def get_wordnet_pos(word):
    '''
    Convert the PennTreebank tags to simple Wordnet tags
    '''
    tag = pos_tag([word])[0][1][0].upper()

    tag_dict = {"J": wn.ADJ,
                "N": wn.NOUN,
                "V": wn.VERB,
                "R": wn.ADV}

    return tag_dict.get(tag, wn.NOUN)

def get_hypernyms(text):

    hypernymn_tweet = []
    
    count = len(text)

    for word in text:
        synsets = wn.synsets(word)

        if len(synsets) <= 0:
            hypernymn_tweet.append(word)
        else:
        # We get the hypernym
            hyper = synsets[0].hypernyms()
        # If we get a result
            if len(hyper) <= 0:
                hypernymn_tweet.append(word)
            else:
                lemmas = hyper[0].lemma_names()
                hypernymn_tweet.append(lemmas[0])
    return hypernymn_tweet
hypernym_text = []
tweets = processed_df['clean_text']
for tweet in tweets:
    hypernym_text.append(get_hypernyms(tweet))
processed_df['hypernym_text'] = hypernym_text
processed_df.head()<jupyter_output><empty_output><jupyter_text>Word Embedding and Reaplcing words with centroid of the clusters of word vectors<jupyter_code># Import word2vec model
from gensim.models import Word2Vec, KeyedVectors
model = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin.gz', binary=True)
model.vector_size
#generate a set of unique tokens in entire corpus
unique_tokens = set()
for text in processed_df['clean_text']:
    unique_tokens.update(text)
print("Found {} unique tokens in corpus".format(len(unique_tokens)))
# Filter the list of vectors to include only those that Word2Vec has a vector for
vector_list = [model[word] for word in unique_tokens if word in model.vocab]

# Create a list of the words corresponding to these vectors
words_filtered = [word for word in unique_tokens if word in model.vocab]

# Zip the words together with their vector representations
word_vec_zip = zip(words_filtered, vector_list)

# Cast to a dict so we can turn it into a DataFrame
word_vec_dict = dict(word_vec_zip)
embeddings_df = pd.DataFrame.from_dict(word_vec_dict, orient='index')
embeddings_df.shape
embeddings_df.head(3)
df = open("embeddings_df.pickle","wb")
pickle.dump(embeddings_df, df)<jupyter_output><empty_output><jupyter_text>Clustering<jupyter_code>from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances_argmin_min
num_of_clusters = len(embeddings_df) // 10

df = embeddings_df

#clustering
    
kmeans = KMeans(n_clusters = num_of_clusters, random_state=42).fit(df)
group_num = kmeans.labels_
centroids = kmeans.cluster_centers_

#assign nearest word to geometric centroid in embedding space as centroid
# find the index of the closest points from x to each class centroid
close = pairwise_distances_argmin_min(centroids, df, metric='euclidean')
index_closest_points = close[0]
word_centroids = df.iloc[index_closest_points].index

#create dict of group number and centroids
centroid_dict = {}
for i in range(len(index_closest_points)):
    centroid_dict[i] = word_centroids[i]

#create a dictionary of word and corresponding centroid

#replace each label(group number) assigned by kmeans cluster algo with centroid word
cen = [centroid_dict.get(group) for group in group_num]

#create a dictionary
word_centroid_dict = {}

for i in range(df.shape[0]):
    word_centroid_dict[df.index[i]] = cen[i]
# Replacing words with the centroid of the cluster
replaced_tokens = []

for tweet in processed_df['clean_text']:
    replaced_tokens.append([word_centroid_dict.get(word) if word in word_centroid_dict else word for word in tweet])
processed_df['word_embedding_text'] = replaced_tokens
processed_df.head()<jupyter_output><empty_output><jupyter_text>List of unique words, along with their cluster centroids and categories.<jupyter_code># categories = ["climate", "world", "state", "member", "globe", "recycling", "money", "future", "strategy", "action"]
categories = ['weather', 'Pollution', 'Policy', 'Action', 'Environment', 'Plastic', 'Initiatives', 'Recycle', 'Earth', 'People']
# assign each unique token to a category from the above list
words_to_categories = {}
for word in unique_tokens:
    similarities = []
    for c in categories:
        if(word in model.vocab):
            similarities.append((c, model.similarity(c, word)))
        else:
            similarities.append((word, 0))
    words_to_categories[word] = sorted(similarities, key = lambda x: x[1], reverse=True)[0][0]
# assign each unique token with its centroid word form the corresponding cluster
words_to_centroids = {}
for word in unique_tokens:
    words_to_centroids[word] = word_centroid_dict.get(word) or word
repalced_words = pd.DataFrame()
repalced_words['words'] = words_to_centroids.keys()
repalced_words['centroids'] = words_to_centroids.values()
repalced_words['categories'] = words_to_categories.values()
repalced_words.head()
repalced_words.to_csv("repalced_words.csv", index=False, header = True)
processed_df.head()<jupyter_output><empty_output><jupyter_text>Score Generation<jupyter_code>def get_wordnet_pos(word):
    '''
    Convert the PennTreebank tags to simple Wordnet tags
    '''
    tag = pos_tag([word])[0][1][0].upper()

    tag_dict = {"J": wn.ADJ,
                "N": wn.NOUN,
                "V": wn.VERB,
                "R": wn.ADV}

    return tag_dict.get(tag, wn.NOUN)


def sentimentAnalyser(text):
    '''
    This Function generates normalized sentiment score and a sentiment category to the tweet.
        Score is normalized for fairer comparison.
    '''
    # to count no. of words which bear scores
    num_of_words = 0
    sentiment_score = 0
    final_score = 0

    # sent_token = sent_tokenize(text)
    word_token = [word_tokenize(sent) for sent in text]
    word_pos_tag = [pos_tag(word) for word in word_token]

    for i in range(len(word_pos_tag)):
        for word, tag in (word_pos_tag[i]):
            wn_tag = get_wordnet_pos(tag)

            synsets = wn.synsets(word, pos=wn_tag)

            if not synsets:
                continue

            word_sent_score = 0
            synset_count = 0
            # Calculate the aveage sentiment score of first 5 synsets of given word
            for synset in synsets:
                synset_count += 1
                swn_synset = swn.senti_synset(synset.name())
                word_sent_score += swn_synset.pos_score() - swn_synset.neg_score()
                num_of_words += 1
                if(synset_count==5):
                    continue

            sentiment_score += word_sent_score
    if(num_of_words==0):
      normalized_score = 0
    else:
      normalized_score = round(sentiment_score/num_of_words, 3)

    # Classify reviews according to setiment score assigned
    # 1 : positive, 0 : neutral, -1 : negative
    if normalized_score > 0:
        final_score = 1
    elif normalized_score < 0:
        final_score = -1
    elif normalized_score == 0:
        final_score = 0

    return final_score


if __name__ == "__main__":
    processed_df
    processed_tweets = processed_df['clean_text']
    scores = []
    for tweet in processed_tweets:
        scores.append(sentimentAnalyser(tweet))
    processed_df['score'] = scores
    # tweet = '''next establish question use question mark liberally indicate question think face indicate not understand 
    # say question compose 3040 communication critical emoji discovery
    # '''
    # tweet_with_score = sentimentAnalyser(tweet)
    # print(tweet_with_score)
    processed_word_embedding_tweets = processed_df['word_embedding_text']
    word_embedding_scores = []
    for tweet in processed_word_embedding_tweets:
        word_embedding_scores.append(sentimentAnalyser(tweet))
    processed_df['word_embedding_scores'] = word_embedding_scores

    processed_hypernymn_tweets = processed_df['hypernym_text']
    hypernymn_scores = []
    for tweet in processed_hypernymn_tweets:
        hypernymn_scores.append(sentimentAnalyser(tweet))
    processed_df['hypernymn_scores'] = hypernymn_scores

    print("Executed Successfully")
processed_df.head()
final_df = open("final_df.pickle","wb")
pickle.dump(processed_df, final_df)
final_df = open("final_df.pickle", "rb")
processed_df = pickle.load(final_df)
processed_df.to_csv("results.csv", index=False)
for n in range(0,100,20):
    print("date:", processed_df["date"][n] )
    print("original_tweet:", processed_df["text"][n])
    print("processed_tweet:", processed_df["clean_text"][n])
    print("replaced_centroids_tweet:", processed_df["word_embedding_text"][n])
    print("hypernymn_text:", processed_df["hypernym_text"][n])
    print("sentiment_score:", processed_df["score"][n])
    print("sentiment_score_with_embeddings:", processed_df["word_embedding_scores"][n])
    print("hypernymn_scores:", processed_df["hypernymn_scores"][n],'\n')
length = len(processed_df)
count = 0
for i in range(len(processed_df)):
    if(processed_df['score'][i] != processed_df['word_embedding_scores'][i]):
        count+=1
print("comparing clean text scores with word embeding scores")
print(" total tweets: {}\n same score: {}\n different score: {}\n".format(length, length - count, count))

hypernynm_count = 0
for i in range(len(processed_df)):
    if(processed_df['score'][i] != processed_df['hypernymn_scores'][i]):
        hypernynm_count+=1
print("comparing clean text scores with hypernymn scores")
print(" total tweets: {}\n same score: {}\n different score: {}".format(length, length - hypernynm_count, hypernynm_count))
positive_count_1 = processed_df[processed_df['score']==1]['score'].count()
positive_count_2 = processed_df[processed_df['hypernymn_scores']==1]['hypernymn_scores'].count()
positive_count_3 = processed_df[processed_df['word_embedding_scores']==1]['word_embedding_scores'].count()

negative_count_1 = processed_df[processed_df['score']==-1]['score'].count()
negative_count_2 = processed_df[processed_df['hypernymn_scores']==-1]['hypernymn_scores'].count()
negative_count_3 = processed_df[processed_df['word_embedding_scores']==-1]['word_embedding_scores'].count()

neutral_count_1 = processed_df[processed_df['score']==0]['score'].count()
neutral_count_2 = processed_df[processed_df['hypernymn_scores']==0]['hypernymn_scores'].count()
neutral_count_3 = processed_df[processed_df['word_embedding_scores']==0]['word_embedding_scores'].count()
# sns bar plot
senti_score_df = pd.DataFrame([[positive_count_1, negative_count_1, neutral_count_1], 
                               [positive_count_2, negative_count_2, neutral_count_2],
                               [positive_count_3, negative_count_3, neutral_count_3]
                               ], 
                              index = ['Clean Text', 'Hypernyms', 'Word Vectors'],
                              columns=['Positive', 'Negative', 'Neutral'])
fig, ax = plt.subplots(figsize=(10,6))
senti_score_df.plot(kind='bar' ,ax=ax)
plt.xticks(rotation=0)
plt.tight_layout()
plt.savefig('score_count.png')<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>pip install adjustText
from sklearn.manifold import TSNE
import adjustText
df = open("embeddings_df.pickle", "rb")
embeddings_df = pickle.load(df)
# Initialize t-SNE
tsne = TSNE(n_components = 2, init = 'random', random_state = 42, perplexity = 100)

# Use only 400 rows to shorten processing time
tsne_df = tsne.fit_transform(embeddings_df[:400])
sns.set()
# Initialize figure
fig, ax = plt.subplots(figsize = (11.7, 8.27))
sns.scatterplot(tsne_df[:, 0], tsne_df[:, 1], alpha = 0.5)

# Import adjustText, initialize list of texts
from adjustText import adjust_text
texts = []
words_to_plot = list(np.arange(0, 400, 10))

# Append words to list
for word in words_to_plot:
    texts.append(plt.text(tsne_df[word, 0], tsne_df[word, 1], embeddings_df.index[word], fontsize = 14))
    
# Plot text using adjust_text (because overlapping text is hard to read)
adjust_text(texts, force_points = 0.4, force_text = 0.4, 
            expand_points = (2,1), expand_text = (1,2),
            arrowprops = dict(arrowstyle = "-", color = 'black', lw = 0.5))

plt.savefig("word_embedding_visualization.png")
plt.show()
from wordcloud import WordCloud
all_words = " ".join([" ".join(text) for text in processed_df['clean_text']])

# text = " ".join(processed_df['clean_text'][0])

# Create and generate a word cloud image:
# wordcloud = WordCloud().generate(text)
wordcloud = WordCloud(width=800, height=500, random_state=42, max_font_size=110).generate(all_words)

# Display the generated image:
plt.figure(figsize=(10, 7))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis('off')
plt.savefig('word_cloud.png')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13829355414840072, 'fraction_numerical': 0.013232667640850787, 'mean_word_length': 3.679392212725546, 'pattern_counts': {'":': 136, '<': 50, '<?xml version=': 0, '>': 49, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '445411', 'license': 'Apache-2.0', 'n_tokens_mistral': 8652, 'n_tokens_neox': 8138, 'n_words': 1875, 'path': 'Advanced_Project_1.ipynb', 'repo_name': 'Narendra1508/Advanced-Project-1', 'size': '24636'}