<jupyter_start><jupyter_text>Corpus Analysis The corpus includes PDF resources provided by <NAME> and collected by <NAME>. There is about 170 articles. This analysis is intended for exploring the corpus we already have collected and to draw insights into finding additional related resources.<jupyter_code>import re
import nltk
import os
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup      
from tika import parser
from nltk import ngrams
from wordcloud import WordCloud
from nltk.corpus import stopwords
from collections import Counter
from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer, CountVectorizer
from sklearn.cluster import KMeans
from sklearn.pipeline import make_pipeline
import matplotlib.pyplot as plt
#plt.style.use('dark_background')
plt.style.use('fivethirtyeight')
%matplotlib notebook<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>raw_corpus_list = os.listdir('../data/pdf/')<jupyter_output><empty_output><jupyter_text>Parsing PDFs<jupyter_code>raw_corpus = ''
for file in raw_corpus_list:
    if file.endswith('pdf'):
        try:
            text = parser.from_file('../data/pdf/' + file)
            raw_corpus = raw_corpus + text['content']
            print('O - ' + str(file))
        except Exception as e:
            print('X - ' + str(e) + ' --- ' + str(file))
print('\n\nThe raw corpus string contains a string character length of: ' + str(len(raw_corpus)))<jupyter_output>O - Ahearn_Dahlgren_2005_Limnology and Oceanography.pdf
O - Amos_2008_Thesis.pdf
O - Ashley et al._2006_Environmental Monitoring and Assessment.pdf
O - Berthelote_2013_Thesis.pdf
O - Bountry_etal_2013_Reviews in Engineering Geology.pdf
O - Bowman_2001_Thesis.pdf
O - Brazell_Cole_2009_Journal of the Pennsylvania Academy of Science.pdf
O - Bulak_etal_2011_Report.pdf
O - Burdick,_Hightower_2006_Transactions of the American Fisheries Society.pdf
O - Burroughs_2003_Thesis.pdf
O - Burroughs_2007_Thesis_Ch2.pdf
O - Burroughs_2007_Thesis_Ch3.pdf
O - Burroughs_etal_2009_Geomorphology.pdf
O - Burroughs_etal_2010_TransactionsOfTheAmericanFisheriesSociety.pdf
X - 'latin-1' codec can't encode character '\u2010' in position 27: ordinal not in range(256) --- Bushaw‐Newton_etal_2003_JAWRA.pdf
O - Cannatelli_Curran_2012_Journal of Hydraulic Engineering.pdf
O - Cantwell_etal_2014_Environmental monitoring and assessment.pdf
O - Casper_etal_2006_Archiv fuer Hydrobiologie Supplement.pdf
O - Catalano_2002_T[...]<jupyter_text>Raw Text Analysis  N-grams<jupyter_code>n = 1
unigrams = ngrams(raw_corpus.split(), n)

ct_unigrams = Counter(unigrams).most_common(20)

vals = []
labels = []
for i in ct_unigrams:
    vals.append(i[1])
    labels.append(i[0][0])
    
fig, axes = plt.subplots()

y_pos = np.arange(len(labels))
plt.bar(y_pos, vals, alpha=0.5)
axes.set_xticks(y_pos)
axes.set_xticklabels(labels, rotation=75)
plt.title('Top 20 Unigrams')
plt.xlabel('Unigrams')
plt.ylabel('Count')
plt.tight_layout()
plt.show()
# Bigrams
n = 2
bigrams = ngrams(raw_corpus.split(), n)

ct_bigrams = Counter(bigrams).most_common(20)

vals = []
labels = []
for i in ct_bigrams:
    vals.append(i[1])
    labels.append(' '.join((i[0])))
    
fig, axes = plt.subplots()

y_pos = np.arange(len(labels))
plt.bar(y_pos, vals, alpha=0.5)
axes.set_xticks(y_pos)
axes.set_xticklabels(labels, rotation=75)
plt.title('Top 20 Bigrams')
plt.xlabel('Bigrams')
plt.ylabel('Count')
plt.tight_layout()
plt.show()
# Trigrams
n = 3
trigrams = ngrams(raw_corpus.split(), n)

ct_trigrams = Counter(trigrams).most_common(20)

vals = []
labels = []
for i in ct_trigrams:
    vals.append(i[1])
    labels.append(' '.join((i[0])))
    
fig, axes = plt.subplots()

y_pos = np.arange(len(labels))
plt.bar(y_pos, vals, alpha=0.5)
axes.set_xticks(y_pos)
axes.set_xticklabels(labels, rotation=80)
plt.title('Top 20 Trigrams')
plt.xlabel('Trigrams')
plt.ylabel('Count')
plt.tight_layout()
plt.show()
# quadgrams -- DO NOT USE
n = 4
quadgrams = ngrams(raw_corpus.split(), n)

ct_quadgrams = Counter(quadgrams).most_common(30)

vals = []
labels = []
for i in ct_quadgrams:
    vals.append(i[1])
    labels.append(' '.join((i[0])))
    
fig, axes = plt.subplots()

y_pos = np.arange(len(labels))
plt.bar(y_pos, vals, alpha=0.5)
axes.set_xticks(y_pos)
axes.set_xticklabels(labels, rotation=90)
plt.title('Top 20 Quadgrams')
plt.xlabel('Quadgrams')
plt.ylabel('Count')
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Pre-Processing<jupyter_code>def review_to_words(raw_review):
    # rm non-letters        
    letters_only = re.sub("[^a-zA-Z]", " ", raw_review) 
    # to lower case, split into individual words
    words = letters_only.lower().split()                             
    # searching a set is faster than searching a list - convert stopwords to a set
    stops = set(stopwords.words("english"))                  
    # rm stop words
    meaningful_words = [w for w in words if not w in stops]   
    # Join the words back into one string separated by space, and return the result.
    return(" ".join(meaningful_words))
# process corpus
clean_corpus = review_to_words(raw_corpus)<jupyter_output><empty_output><jupyter_text>Save Cleaned Corpus<jupyter_code>with open('clean_full_corpus.txt', 'w') as f:
    f.write(clean_corpus)<jupyter_output><empty_output><jupyter_text>Create Features<jupyter_code>vectorizer = CountVectorizer(analyzer = "word", 
                             tokenizer = None, 
                             preprocessor = None, 
                             stop_words = None, 
                             max_features = 300) 
train_data_features = vectorizer.fit_transform(clean_corpus.split('water'))

train_data_features = train_data_features.toarray()
train_data_features.shape<jupyter_output><empty_output><jupyter_text>Vocabulary<jupyter_code>vocab = vectorizer.get_feature_names()
print(vocab)
# Sum up the counts of each vocabulary word
dist = np.sum(train_data_features, axis=0)

counts = []
for tag, count in zip(vocab, dist):
    tup = (count, tag)
    counts.append(tup)
    print(count, tag)
top_20 = sorted(counts)[-20:]
print(top_20)
vals = []
labels = []
for i in top_20:
    vals.append(i[0])
    labels.append(i[1])
fig, axes = plt.subplots()

y_pos = np.arange(len(labels))
plt.bar(y_pos, vals, alpha=0.5)
axes.set_xticks(y_pos)
axes.set_xticklabels(labels, rotation=90)
plt.title('Top 20 Terms')
plt.xlabel('Terms')
plt.ylabel('Count')
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Ngrams<jupyter_code># Bigrams
n = 2
bigrams = ngrams(clean_corpus.split(), n)

ct_bigrams = Counter(bigrams).most_common(20)

vals = []
labels = []
for i in ct_bigrams:
    vals.append(i[1])
    labels.append(' '.join((i[0])))
    
fig, axes = plt.subplots()

y_pos = np.arange(len(labels))
plt.bar(y_pos, vals, alpha=0.5)
axes.set_xticks(y_pos)
axes.set_xticklabels(labels, rotation=90)
plt.title('Top 20 Bigrams')
plt.xlabel('Bigrams')
plt.ylabel('Count')
plt.tight_layout()
plt.show()
len(clean_corpus.split())
# Trigrams
n = 3
trigrams = ngrams(clean_corpus.split(), n)

ct_trigrams = Counter(trigrams).most_common(20)

vals = []
labels = []
for i in ct_trigrams:
    vals.append(i[1])
    labels.append(' '.join((i[0])))
    
fig, axes = plt.subplots()

y_pos = np.arange(len(labels))
plt.bar(y_pos, vals, alpha=0.5)
axes.set_xticks(y_pos)
axes.set_xticklabels(labels, rotation=90)
plt.title('Top 20 Trigrams')
plt.xlabel('Trigrams')
plt.ylabel('Count')
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Word Cloud<jupyter_code>wordcloud = WordCloud(width=1000, height=500, margin=10, font_path='Helvetica').generate(clean_corpus)
fig, axes = plt.subplots()
axes.grid(False)
axes.get_xaxis().set_visible(False)
axes.get_yaxis().set_visible(False)

plt.imshow(wordcloud, interpolation="bilinear")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1282149223325694, 'fraction_numerical': 0.02393684746625923, 'mean_word_length': 4.779985283296542, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '275026', 'license': 'CC0-1.0', 'n_tokens_mistral': 3159, 'n_tokens_neox': 2857, 'n_words': 584, 'path': 'notebooks/corpus_analysis.ipynb', 'repo_name': 'bserna-usgs/dam_removal_analysis', 'size': '7867'}