<jupyter_start><jupyter_text>Word and PDF Documents  Word documents- textract- python-docx- docx2text<jupyter_code>from glob import glob
word_files = glob('data/gfsr_docs/docx/*.docx')
word_files<jupyter_output><empty_output><jupyter_text>textract with .docx files<jupyter_code>import textract
text = textract.process(word_files[0])
print(type(text))
print(text[:10])
text = text.decode('utf-8')
print(type(text))
text[:200]<jupyter_output><class 'bytes'>
b'Report on '
<class 'str'><jupyter_text>Check encodings.<jupyter_code>from bs4 import UnicodeDammit

with open(word_files[0], 'rb') as f:
    blob = f.read()
    suggestion = UnicodeDammit(blob)
    print(suggestion.original_encoding)<jupyter_output>Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.<jupyter_text>Remove punctuation and numbers.<jupyter_code>import string

translator = str.maketrans('', '', string.punctuation + string.digits)
text = text.translate(translator)
text[:100]
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords

en_stopwords = stopwords.words('english')

print(en_stopwords[:10])

en_stopwords = set(en_stopwords)
words = text.lower().split()
words[:10]
words = [w for w in words if w not in en_stopwords and len(w) > 3]
words[:10]
bigrams = list([' '.join(bg) for bg in nltk.bigrams(words)])
bigrams[:3]
ug_fdist = nltk.FreqDist(words)
bg_fdist = nltk.FreqDist(bigrams)
ug_fdist.most_common(20)
bg_fdist.most_common(20)
import matplotlib.pyplot as plt

ax = ug_fdist.plot(20)
bg_fdist.plot(20)
from wordcloud import WordCloud

plt.figure(figsize=(5.5, 5.5))
wordcloud = WordCloud(collocations=False, height=300, width=300, scale=3).generate(' '.join(words))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Let's create a function which will take a path to a folder and generate unigram and bigram frequency plots.<jupyter_code>import os
from glob import glob
import textract
import nltk

en_stopwords = set(nltk.corpus.stopwords.words('english'))


def create_fdist_visualizations(path):
    """
    Takes a path to a folder with .docx files, reads and cleans text,
    then plots unigram and bigram frequency distributions.
    """
    word_docs = glob(os.path.join(path, '*.docx'))
    text = ' '.join([textract.process(w).decode('utf-8') for w in word_docs])
    
    # remove punctuation, numbers, stopwords
    translator = str.maketrans('', '', string.punctuation + string.digits)
    text = text.translate(translator)
    words = text.lower().split()
    words = [w for w in words if w not in en_stopwords and len(w) > 3]
    
    unigram_fd = nltk.FreqDist(words)
    bigrams = list([' '.join(bg) for bg in nltk.bigrams(words)])
    bigram_fd = nltk.FreqDist(bigrams)
    
    unigram_fd.plot(20)
    bigram_fd.plot(20)
create_fdist_visualizations('data/gfsr_docs/docx/')<jupyter_output><empty_output><jupyter_text>.doc filestextract can also parse .doc files (an older Word format), but requires a software called 'antiword'. This is available [here](https://www.softpedia.com/get/Office-tools/Other-Office-Tools/Antiword.shtml) for Windows (instructions [here](https://stackoverflow.com/a/65982286/4549682)), and the [installation instructions](https://textract.readthedocs.io/en/stable/installation.html) for textract show how to install for Mac or Ubuntu/Debian systems.<jupyter_code>doc_text = textract.process('data/gfsr_docs/gfsa03-04rpt.doc')
doc_text = doc_text.decode('utf-8')
doc_text[:100]<jupyter_output><empty_output><jupyter_text>We can check the encoding with a few different libraries - in this case, it looks to be windows-1252.<jupyter_code>with open('data/gfsr_docs/gfsa03-04rpt.doc', 'rb') as f:
    blob = f.read()
    suggestion = UnicodeDammit(blob)

print(suggestion.original_encoding)<jupyter_output>Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.<jupyter_text>python-docxWe can use python-docx as well, though there is an extra step to get the text:<jupyter_code>import docx
doc = docx.Document(word_files[0])
text = ' '.join([p.text for p in doc.paragraphs])
words = text.split()
words[:10]<jupyter_output><empty_output><jupyter_text>PDF files<jupyter_code>pdf_files = glob('data/ds_pdfs/*.pdf')
pdf_files<jupyter_output><empty_output><jupyter_text>Again, we can check the encoding of the file if we want.<jupyter_code>with open(pdf_files[0], 'rb') as f:
    blob = f.read()
    suggestion = UnicodeDammit(blob)

print(suggestion.original_encoding)<jupyter_output>Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.<jupyter_text>We need to install the pdfminer library for one way to read pdfs: `conda install -c conda-forge pdfminer-six`.<jupyter_code>from pdfminer.high_level import extract_text
text = extract_text(pdf_files[0])
print(text[:50])<jupyter_output>ENABLING COLLABORATIVE DATA SCIENCE DEVELOPMENT WI<jupyter_text>For better preprocessing where we keep all hyphenated words over newlines, we might do something like this: https://stackoverflow.com/a/43667304/4549682<jupyter_code>lines = text.split('\n')
cleaned_lines = []
for ln in lines:
    if len(ln) == 0:
        continue
    if ln[-1] == '-':
        cleaned_lines.append(ln[:-1])
    else:
        cleaned_lines.append(ln + ' ')

cleaned = ''.join(cleaned_lines)
cleaned[:200]
def create_fdist_visualizations(path, extension='docx'):
    """
    Takes a path to a folder with .docx files, reads and cleans text,
    then plots unigram and bigram frequency distributions.
    """
    docs = glob(os.path.join(path, f'*.{extension}'))
    if extension in['doc', 'docx']:
        text = ' '.join(textract.process(w).decode('utf-8') for w in docs)
    elif extension == 'pdf':
        text = ' '.join(extract_text(w) for w in docs)
        lines = text.split('\n')
        cleaned_lines = []
        for l in lines:
            if len(l) == 0:
                continue
            if l[-1] == '-':
                cleaned_lines.append(l[:-1])
            else:
                cleaned_lines.append(l + ' ')

        text = ''.join(cleaned_lines)
    
    # remove punctuation, numbers, stopwords
    translator = str.maketrans('', '', string.punctuation + string.digits)
    text = text.translate(translator)
    words = text.lower().split()
    words = [w for w in words if w not in en_stopwords and len(w) > 3]
    
    unigram_fd = nltk.FreqDist(words)
    bigrams = list([' '.join(bg) for bg in nltk.bigrams(words)])
    bigram_fd = nltk.FreqDist(bigrams)
    
    unigram_fd.plot(20)
    bigram_fd.plot(20)
create_fdist_visualizations('data/ds_pdfs/', extension='pdf')<jupyter_output><empty_output><jupyter_text>Other PDF reading libraries If you don't have JDK installed, you need to install JDK separately or do `conda install -c conda-forge openjdk`. `conda install -c conda-forge tika -y`<jupyter_code>from tika import parser
text = parser.from_file(pdf_files[0])
# returns a dictionary
print(text.keys())
print(text['content'].strip()[:200])<jupyter_output>dict_keys(['metadata', 'content', 'status'])
Enabling collaborative data science development with the Ballet framework


ENABLING COLLABORATIVE DATA SCIENCE DEVELOPMENT WITH THE
BALLET FRAMEWORK

<NAME> 1 <NAME> 2 3 <NAME> 1 Kal<jupyter_text>`pip install pymupdf`<jupyter_code>import fitz  # pymupdf

with fitz.open(pdf_files[0]) as doc:
    text = ""
    for page in doc:
        text += page.getText()

print(text[:200])<jupyter_output>ENABLING COLLABORATIVE DATA SCIENCE DEVELOPMENT WITH THE
BALLET FRAMEWORK
<NAME> 1 <NAME> 2 3 <NAME> 1 <NAME> 1
ABSTRACT
While the open-source model for software develo
{'fraction_non_alphanumeric': 0.12260086173129651, 'fraction_numerical': 0.017104060582321452, 'mean_word_length': 4.733532934131737, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 62, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '337722', 'license': 'MIT', 'n_tokens_mistral': 2796, 'n_tokens_neox': 2646, 'n_words': 650, 'path': '6-Chapter-6/Chapter_6_code-Word_and_PDFs.ipynb', 'repo_name': 'DiegoMerino28/Practical-Data-Science-with-Python', 'size': '7708'}