<jupyter_start><jupyter_text>Basic NLP and text analysis on article "A Complexity Measure" by <NAME>.<jupyter_code>from google.colab import drive
drive.mount("/content/gdrive/", force_remount=True)
!ln -s /content/gdrive/My\ Drive/ /content/g
gdrive = "/content/g/"
import pandas as pd
import operator
from pathlib import Path
from nltk import sent_tokenize, word_tokenize
from functools import reduce
from collections import Counter
from unicodedata import normalize

from nltk.corpus import stopwords
stoplist = set(stopwords.words('english'))

from string import punctuation
stoplist = stoplist.union(set(punctuation))

from functools import partial
normalizer = partial(normalize, 'NFKD')

from nltk.stem import PorterStemmer
porter = PorterStemmer()

import nltk
nltk.download('all')
article = Path('/content/gdrive/My Drive/Documents/Read/mccabe.pdf.txt').read_text(errors='ignore')
words = map(porter.stem, map(str.casefold, map(normalizer, reduce(operator.iadd, map(word_tokenize, reduce(operator.iadd, map(sent_tokenize, article.split('\n'))))))))
words = Counter({_[0]: _[1] for _ in 
                 Counter(words).most_common() if _[0] not in stoplist and _[0].isalpha() and len(_[0]) > 2 and _[1] > 2}).most_common()
words
from os import path
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS
wordcloud = WordCloud().generate_from_frequencies(dict(words))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
# Tokenize the article into sentences: sentences
sentences = nltk.sent_tokenize(article)

# Tokenize each sentence into words: token_sentences
token_sentences = [word_tokenize(sent) for sent in sentences]

# Tag each tokenized sentence into parts of speech: pos_sentences
pos_sentences = [nltk.pos_tag(sent) for sent in token_sentences] 

# Create the named entity chunks: chunked_sentences
chunked_sentences = nltk.ne_chunk_sents(pos_sentences, binary=True)

# Test for stems of the tree with 'NE' tags
for sent in chunked_sentences:
    for chunk in sent:
        if hasattr(chunk, "label") and chunk.label() == "NE":
            print(chunk)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10441582336706531, 'fraction_numerical': 0.0036798528058877645, 'mean_word_length': 5.651376146788991, 'pattern_counts': {'":': 1, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '459558', 'license': 'Unlicense', 'n_tokens_mistral': 712, 'n_tokens_neox': 691, 'n_words': 205, 'path': 'nlp_mccabe.ipynb', 'repo_name': 'brl0/brl0-stuff', 'size': '2184'}