<jupyter_start><jupyter_text>**TAD_Week_4_Assignment_Broker_Carl**:---  Part One:  Setup<jupyter_code># load packages
import re
import PyPDF2
import pandas as pd<jupyter_output><empty_output><jupyter_text>1. Which Tweets from Nuclear Sentiment dataset mention Japan.<jupyter_code># mention japan function
def mention_japan(x):
    x = pd.read_csv(x)
    pattern = 'Japan'# set pattern
    output = x[x.tweet_text.str.contains(pattern, case=False)] #subset df w/tweets containing pattern
    return output

mention_japan('sentiment_nuclear_power _1_.csv')<jupyter_output><empty_output><jupyter_text>2. Find all words starting with an upper case in Supreme Court case from Week 3 using RegEx.<jupyter_code># function that counts upper case items in a pdf
def upp_func(x):
    upp_count = 0
    pdfFileObj = open(x, 'rb')
    pdfReader = PyPDF2.PdfFileReader(pdfFileObj)
    l = pdfReader.numPages
    
    for item in range(0,l,1):
        pageObj = pdfReader.getPage(item)
        temp_doc = pageObj.extractText()
        temp_doc = temp_doc.replace('\n', '')
        #print(temp_doc)
        for word in temp_doc:
            if word[0].isupper() == True:
                upp_count += 1
    return upp_count    
# Brown_Warren_Opinion.pdf upp_func
upp_func('Brown_Warren_Opinion.pdf')
# Miranda_Warren_Opinion.pdf upp_func
upp_func('Miranda_Warren_Opinion.pdf')
# Y<NAME> v. California - Justice Matthews opinion.pdf upp_func
upp_func('Yick Wo v. California - Justice Matthews opinion.pdf')
# New York Times v. U.S. - Per Curiam.pdf upp_func
upp_func('New York Times v. U.S. - Per Curiam.pdf')
# Plessy v. Ferguson - Justice Brown opinion.pdf upp_func
upp_func('Plessy v. Ferguson - Justice Brown opinion.pdf')
# Regents of the Univ. of CA v Bakke - Plurality Opinion by Justice Powell.pdf upp_func
upp_func('Regents of the Univ. of CA v Bakke - Plurality Opinion by Justice Powell.pdf')<jupyter_output><empty_output><jupyter_text>3. Remove all occurrences of Twitter usernames (@) and hashtags () from the Nuclear Sentiment dataset from Week 3<jupyter_code># clean twitter csv of '@' and '#' containing substrings

def regex_magic(tweet):
    tweet = ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",tweet).split())
    return tweet

def clean_twitter(csv):
    df_nuke = pd.read_csv(csv)
    df_nuke['tweet_text'] = df_nuke['tweet_text'].apply(regex_magic)
    return df_nuke
    
clean_twitter('sentiment_nuclear_power _1_.csv')<jupyter_output><empty_output><jupyter_text>4. Remove the (fake and inserted by your instructor) social security numbers in the Amazon Musical Instruments JSON ATTACHED TO THIS WEEK'S HOMEWORK.<jupyter_code># clean json of SSNs

def regex_magic(review):
    review = ' '.join(re.sub(r'^.*?(\d{3}-\d{2}-\d{4}).*$'," ",review).split())
    return review

def regex_magic_2(review):
    review = ' '.join(re.sub(r'^.*?(\d{3}-\d{3}-\d{4}).*$'," ",review).split())
    return review

def clean_music(json):
    df_music = pd.read_json(json, lines=True)
    df_music['reviewText'] = df_music['reviewText'].apply(regex_magic) #removes SSNs ###-##-####
    df_music['reviewText'] = df_music['reviewText'].apply(regex_magic) #removes SSNs ###-###-####
    return df_music

clean_music('Musical_Instruments_5_SSN.json')<jupyter_output><empty_output><jupyter_text>5. Fix the attached Nuclear Sentiment CSV. There is an error(s) somewhere.... that will stop R from reading it in completely.<jupyter_code># in python the error does not occur
df_nuke = pd.read_csv('sentiment_nuclear_power _1_.csv')
# inspect
df_nuke<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13082287308228732, 'fraction_numerical': 0.008368200836820083, 'mean_word_length': 4.533950617283951, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '424133', 'license': 'MIT', 'n_tokens_mistral': 1313, 'n_tokens_neox': 1240, 'n_words': 308, 'path': 'notebooks/TAD_Week_4_Broker_Carl.ipynb', 'repo_name': 'cbroker1/text-as-data', 'size': '3585'}