<jupyter_start><jupyter_text>Table of Contents<jupyter_code>import pandas as pd
from bs4 import BeautifulSoup
import numpy as np
import os
def read_function(filename):
    ll = []
    with open(filename) as f:
        contents = f.read()
    contents = contents.split('<FILENAME>')[1]
    s = BeautifulSoup(contents, "html.parser")
    f = s.find_all('p')
    if len(f)<5:
        f = s.find_all('div')

    for i in range(len(f)):
        tt = f[i].text
        if tt is not None:
            ll = np.append(ll,tt)
    result = " ".join(ll).replace(u'\xa0', u' ').replace(u'\n',u' ').replace(u'\t',u' ')
    result = " ".join(result.split())
    
    name = filename.split('_')[0]
    date = filename.split('_')[1].split('.')[0]
    
    print('Done: '+filename)
    
    #return s
    return (name, date, result)
filelist= os.listdir('data10k')
filelist
df =pd.DataFrame(columns=['Name','Date','Story'])
for f in filelist:
    info = read_function('data10k/'+f)
    df = df.append({'Name':info[0],'Date':info[1],"Story":info[2]}, ignore_index=True)
df["Date"]=pd.to_datetime(df["Date"])
df['Name'] = df['Name'].apply(lambda x: x.split('/')[-1])
df.to_csv('finance_data_10k.csv', index=False, encoding='utf8')
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1995172968624296, 'fraction_numerical': 0.014481094127111826, 'mean_word_length': 3.6943396226415093, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '6058', 'license': 'MIT', 'n_tokens_mistral': 477, 'n_tokens_neox': 464, 'n_words': 91, 'path': 'cleaning/data_cleaning_10k.ipynb', 'repo_name': 'SouthernMethodistUniversity/Python_Finance_TextMining', 'size': '1243'}