<jupyter_start><jupyter_text>HW4 -- Sentiment and Lies  STEP 1: Import the dataNOTE: May need to change delimiter based on the data file<jupyter_code>import pandas as pd
df = pd.read_csv('deception_data_converted_final.csv',  sep='\t')
df[:5]<jupyter_output><empty_output><jupyter_text>STEP 2: Pull out the labels<jupyter_code>def get_labels(row):
    split_row = str(row).split(',')
    lie = split_row[0]
    sentiment = split_row[1]
    return [lie, sentiment, split_row[2:]]

df['all'] = df.apply(lambda row: get_labels(row['lie,sentiment,review']), axis=1)
df[:5]
df['lie'] = df.apply(lambda row: row['all'][0][0], axis=1)
df[:5]
df['sentiment'] = df.apply(lambda row: row['all'][1][0], axis=1)
df[:5]
df['review'] = df.apply(lambda row: ''.join(row['all'][2]), axis=1)
df[:5]
clean_df = df.copy()
clean_df.drop(['lie,sentiment,review', 'all'], axis=1, inplace=True)
clean_df<jupyter_output><empty_output><jupyter_text>STEP 3: Clean the data<jupyter_code>def clean_rogue_characters(string):
    exclude = ['\\',"\'",'"']
    string = ''.join(string.split('\\n'))
    string = ''.join(ch for ch in string if ch not in exclude)
    return string

clean_df['review'] = clean_df['review'].apply( lambda x: clean_rogue_characters(x) )
clean_df['review'][0]<jupyter_output><empty_output><jupyter_text>STEP 4: Export cleaned, formatted CSV<jupyter_code>clean_df.to_csv('hw4_data.csv',index=False)
df = pd.read_csv('hw4_data.csv')
df[:5]<jupyter_output><empty_output><jupyter_text>STEP 5: Split df into data sets   LIE DFs<jupyter_code>lie_df_f = df[df['lie'] == 'f']
lie_df_t = df[df['lie'] == 't']<jupyter_output><empty_output><jupyter_text>SENTIMENT DFs<jupyter_code>sent_df_n = df[df['sentiment'] == 'n']
sent_df_p = df[df['sentiment'] == 'p']<jupyter_output><empty_output><jupyter_text>STEP 5b: Export to Corpus to run on current pipelines<jupyter_code>def print_to_file(rating, review, num, title):
    both = review
    output_filename = str(rating) + '_'+ title +'_' + str(num) + '.txt'
    outfile = open(output_filename, 'w')
    outfile.write(both)
    outfile.close()

def export_to_corpus(df, subj, title):
    for num,row in enumerate(df['review']):
        print_to_file(subj, row, num, title)
export_to_corpus(sent_df_n, 'neg', 'hw4_n')
export_to_corpus(sent_df_p, 'pos', 'hw4_p')
export_to_corpus(lie_df_f, 'false', 'hw4_f')
export_to_corpus(lie_df_t, 'true', 'hw4_t')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17661691542288557, 'fraction_numerical': 0.013681592039800995, 'mean_word_length': 6.809061488673139, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '621447', 'license': 'MIT', 'n_tokens_mistral': 986, 'n_tokens_neox': 972, 'n_words': 176, 'path': 'assets/all_html/2019_10_24_HW4.ipynb', 'repo_name': 'dskw1/dskw1.github.io', 'size': '2412'}