<jupyter_start><jupyter_text>Process Data<jupyter_code>#bring in corpus
with open('/Users/user/desktop/metis/projects/metis-project-5/data/data/cbt_train.txt', 'r') as f_in:
    entire = f_in.read()
#split into books
books = entire.split('_BOOK_TITLE_')
#make df
df = pd.DataFrame(books, columns =['Text']) 
df
df.loc[1, 'Text']
#split out text and title 
first_split = df["Text"].str.split(".txt", n = 1, expand = True) 

df["Title"] = first_split[0]

df["Text"]= first_split[1] 
df
#split out title and author
second_split = df["Title"].str.split("__", n = 1, expand = True) 

df["Author"] = second_split[0]

df["Title"]= second_split[1] 
df
#separate by chapters
df = df.assign(Text=df.Text.str.split('CHAPTER')).explode('Text').reset_index(drop=True)
#remove bad rows
df = df[df.Text != '.out\n']

#strip whitespace
df['Text'] = df.Text.str.strip()

#extract chapter and place in new column
df['Chapter'] = df.Text.str.extract("^([IVXL]+)")

#remove chapters now
df['Text'] = df.Text.str.strip("^([IVXL]+)")

#remove new lines
df['Text'] = df.Text.str.strip("\n")
df
def readability_clean(text):
    text = re.sub('\n', '', text) #remove linebreaks
    text = re.sub('`', '', text) #remove weird thing
    text = re.sub(r'\s+([?.!:";,])', r'\1', text) #remove whitespace preceding certain punctuation
    
    return text
df['Text'] = df['Text'].astype(str)
df['Clean_Text'] = df['Text'].apply(readability_clean)
#more cleaning
df['Clean_Text'] = df['Clean_Text'].str.replace('-LCB-', '')
df['Clean_Text'] = df['Clean_Text'].str.replace('Chapter heading picture', '')
df['Clean_Text'] = df['Clean_Text'].str.replace('.jpg', '')
df['Clean_Text'] = df['Clean_Text'].str.replace('-RCB-', '')
df['Clean_Text'] = df['Clean_Text'].str.replace('p1', '')
def more_clean(text):
    text = re.sub("^.*?([A-Z])", "\\1", text) #removes everything before the first capital letter
    text = re.sub(r'\.(?! )', '. ', re.sub(r' +', ' ', text)) # add space after period and compress double spaces
    
    return text
#apply this
df['Read_text'] = df['Clean_Text'].apply(more_clean)
#general DF clean up

#get ride of dirty columns and rows
#df = df.drop(columns=['Text', 'Clean_Text'])
df = df.iloc[1:]

#clean up title column
df['Title'] = df['Title'].str.replace('_', ' ')
df['Title'] = df['Title'].str.lstrip(' ')

#clean up author column
df['Author'] = df['Author'].str.replace('_', ' ')
df['Author'] = df['Author'].str.replace(':', '')

#rename columns
df = df.rename(columns={"slim": "NLP_Text", "Read_text": "Readable_Text"})

#reset index
df = df.reset_index(drop=True)
df
#pickle
df.to_pickle('/Users/user/desktop/metis/projects/metis-project-5/childrens_book_df.pkl')
#clean for new column for NLP stuff

def clean(text):
    #make lowercase, remove links, characters, punctuation, etc.
    text = text.lower()
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('[’“”…]', '', text)
    
    # removing the stop-words          
    text_tokens = word_tokenize(text)
    tokens_without_sw = [word for word in text_tokens if not word in stop_words]
    filtered_sentence = (" ").join(tokens_without_sw)
    text = filtered_sentence
    
    return text
stop_words = stopwords.words()

df['slim'] = df['Text'].apply(clean)
df.iloc[10,4]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21046477638117508, 'fraction_numerical': 0.00467699503069278, 'mean_word_length': 4.619047619047619, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '77421', 'license': 'MIT', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1241, 'n_words': 272, 'path': '01_Cleaning_Pre-Processing.ipynb', 'repo_name': 'mattsegall/metis-project-5', 'size': '3421'}