<jupyter_start><jupyter_text>Loading journal text entries<jupyter_code>users = pd.read_csv('/Users/afrah/Desktop/catchingfeels/data/users.csv')
journals = pd.read_csv('/Users/afrah/Desktop/catchingfeels/data/journals.csv')
journals.head()<jupyter_output><empty_output><jupyter_text>Dropping NAN entries in sentiment/journal/date<jupyter_code>journals_sub = journals.dropna(subset=['sentiment', 'journal', 'date'])
journals_sub['sentiment_level'] = journals_sub[[
    'sentiment'
]].apply(lambda x: pd.cut(x, [-np.inf, -.5, .5, np.inf],
                          labels=['negative', 'neutral', 'positive']))<jupyter_output>/Users/afrah/miniconda3/envs/test/lib/python3.7/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  after removing the cwd from sys.path.<jupyter_text>Feature Engineering Time featuresTime of day, month, seasons<jupyter_code>journals_sub['date'] = pd.to_datetime(journals_sub['date'])
journals_sub['year'] = pd.Series(journals_sub['date'].dt.year,
                                 dtype='category')
journals_sub['day'] = journals_sub['date'].dt.day_name()
journals_sub['month'] = journals_sub['date'].dt.month_name()
journals_sub['month_trans'] = np.cos(
    (journals_sub['date'].dt.month - 6.0) / 6.0)
journals_sub['timeofday'] = journals_sub['date'].dt.hour
journals_sub['timeofday_trans'] = np.cos(
    (journals_sub['date'].dt.hour - 12.0) / 12.0)
plt.figure(figsize=(15, 8))
journals_sub.hist(figsize=(20, 10),
                  ylabelsize=15,
                  xlabelsize=15,
                  xrot=20,
                  bins=20,
                  edgecolor='white')
sns.countplot(x='year',
              hue='sentiment_level',
              data=journals_sub,
              orient='horizontal',
              edgecolor='white')<jupyter_output><empty_output><jupyter_text>Time of day<jupyter_code>plt.figure(figsize=(10, 8))
# plt.subplot(2,2,1)
ax = sns.boxplot(x="sentiment_level", y='timeofday_trans', data=journals_sub)
ax.set_xticklabels(ax.get_xticklabels(), rotation=20)

# plt.subplot(2,2,3)
# plt.scatter(x=np.arange(24), y= np.cos((np.arange(24)-12)/12))
# plt.axhline(y=0.75)
plt.figure(figsize=(15, 15))
plt.subplot(3, 1, 1)
ax = sns.boxplot(x="sentiment_level", y='month_trans', data=journals_sub)
ax.set_xticklabels(ax.get_xticklabels(), rotation=20)

plt.subplot(3, 1, 2)
ax = sns.boxplot(y="sentiment", x='month', data=journals_sub)
ax.set_xticklabels(ax.get_xticklabels(), rotation=20)

plt.subplot(3, 1, 3)
plt.scatter(x=np.arange(12), y=np.cos((np.arange(12) - 6) / 6))<jupyter_output><empty_output><jupyter_text>SeasonsDo Seasons influence mood predictions? Especially in the context of seasonal affective disorders<jupyter_code>Winter = ['December', 'January', 'February']
Spring = ['March', 'April', 'May']
Summer = ['June', 'July', 'August']
Fall = ['September', 'October', 'November']

seasons = dict(
    zip(Winter + Spring + Summer + Fall,
        ['Winter'] * 3 + ['Spring'] * 3 + ['Summer'] * 3 + ['Fall'] * 3))
journals_sub['season'] = journals_sub['month'].map(seasons)

seasons_color = dict(
    zip(Winter + Spring + Summer + Fall,
        ['skyblue'] * 3 + ['lightgreen'] * 3 + ['#FFD43B'] * 3 + ['Red'] * 3))<jupyter_output>/Users/afrah/miniconda3/envs/test/lib/python3.7/site-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  if __name__ == '__main__':<jupyter_text>Winter seems to have the lowest mood predictions; though seem insignificant<jupyter_code>plt.figure(figsize=(15, 8))
ax = sns.boxplot(y="sentiment",
                 x='season',
                 data=journals_sub,
                 palette={
                     'Winter': 'skyblue',
                     'Fall': 'Red',
                     'Spring': 'lightgreen',
                     'Summer': '#FFD43B'
                 })
ax.set_xticklabels(ax.get_xticklabels())<jupyter_output><empty_output><jupyter_text>Content-based features Personal and Non-personal pronouns<jupyter_code>def get_pprn(doc):
    inner_pprn = ('i', 'me', 'my', 'mine', 'myself')
    outer_pprn = ('we', 'you', 'your', 'yourself', 'yours', 'yourselves', 'he',
                  'him', 'his', 'himself', 'she', 'her', 'hers', 'herself',
                  'they', 'them', 'their', 'theirs', 'themselves')
    inner_count = 0
    outer_count = 0

    for token in doc:
        if token.text in inner_pprn:
            inner_count += 1
        elif token.text in outer_pprn:
            outer_count += 1
    return inner_count, outer_count
journals_sub['word_len'] = journals_sub['journal'].apply(lambda x: len(x.split(' ')))
journals_sub['word_len'].plot(kind='hist', logy=True, edgecolor='white')

# print(sum(journals_sub['word_len']>50))
plt.xlabel('# Words in Entry')
nlp = spacy.load("en_core_web_sm")
i_count_all = []
o_count_all = []

for entry in journals_sub['journal']:
    doc = nlp(entry.lower())
    i_count, o_count = get_pprn(doc)
    i_count_all.append(i_count)
    o_count_all.append(o_count)
journals_sub['inner_pprn'] = np.array(i_count_all)
journals_sub['outer_pprn'] = np.array(o_count_all)
journals_sub[
    'sum_pprn'] = journals_sub['outer_pprn'] + journals_sub['inner_pprn']
journals_sub[
    'inner_frac'] = journals_sub['inner_pprn'] / journals_sub['sum_pprn']
journals_sub['inner_frac'] = journals_sub['inner_frac'].fillna(0)
journals_sub[
    'outer_frac'] = journals_sub['outer_pprn'] / journals_sub['sum_pprn']
journals_sub['outer_frac'] = journals_sub['outer_frac'].fillna(0)
journals_sub['pprn_ratio'] = np.array(i_count_all) / (np.array(o_count_all) +
                                                      1)
plt.figure(figsize=(20, 10))
sns.pairplot(corner=True,
             data=journals_sub[[
                 'pprn_ratio', 'inner_frac', 'outer_frac', 'sum_pprn',
                 'sentiment_level'
             ]],
             hue='sentiment_level')
plt.ylabel(['P', 'A', 'B'])
sns.boxplot(x='sentiment_level',
            y='pprn_ratio',
            data=journals_sub[journals_sub['sum_pprn'] > 20])
plt.ylabel('Personal: Non-personal pronouns')
plt.xlabel('')
stop_words = set(stopwords.words('english'))

filtered_sentences = journals_sub['journal'].apply(
    lambda x: ' '.join([i for i in WhitespaceTokenizer().tokenize(x)]))
corpus = journals_sub['journal']
vectorizer = TfidfVectorizer(stop_words=stopwords.words('english'),
                             token_pattern=r'(?u)\b[A-Za-z]+\b')
mat = vectorizer.fit_transform(corpus)<jupyter_output><empty_output><jupyter_text>Readability scores<jupyter_code>journals_sub['polysyllable_count'] = journals_sub['journal'].apply(
    lambda x: textstat.textstat.polysyllabcount(x))
journals_sub['difficult_words'] = journals_sub['journal'].apply(
    lambda x: textstat.textstat.difficult_words(x))
journals_sub['cl_index'] = journals_sub['journal'].apply(
    lambda x: textstat.textstat.coleman_liau_index(x))
sns.pairplot(data=journals_sub[[
    'polysyllable_count', 'difficult_words', 'cl_index', 'word_len'
,'sentiment']],
             corner=True)<jupyter_output><empty_output><jupyter_text>User behavior features  Users-frequency of posting<jupyter_code>journals_sub['user'] = journals_sub['path'].apply(
    lambda x: re.sub('"', '',
                     x.split(',')[1].strip()))
# Number of journal entries varies per user
plt.figure(figsize=(10, 8))
plt.hist(journals_sub['user'].value_counts(), bins=20, edgecolor='white')
plt.yscale('log')
plt.xlabel('# Journal Entries')
plt.ylabel('# Users')
# Variance in sentiment scores may also be associated with # entries by a user
plt.hexbin(y=journals_sub.groupby('user').mean()['sentiment'],
           x=journals_sub['user'].value_counts(),
           gridsize=20,
           cmap='Blues',
           bins='log')
plt.xlabel('# Entries by user')
plt.ylabel('Mean(Sentiment)')
plt.colorbar()
# save all journal entries
# journals_sub.to_csv('../data/all_journals.csv', index=False)<jupyter_output><empty_output><jupyter_text>Audio features<jupyter_code>df = pd.read_csv('/Users/afrah/Desktop/catchingfeels/data/audio_features13030.csv')
sns.pairplot(data=df, corner=True)
journals_sub.columns
# journals_sub = journals_sub[journals_sub['word_len']>=50]
tmp = pd.read_csv('../data/')
tmp.columns
journals_merge = journals_sub[['sentiment', 'date', 'recordingname', 'user',
                              'sentiment_level', 'pprn_ratio', 'sum_pprn',
                              'inner_frac', 'word_len', 'journal',
                              'timeofday_trans', 'season', 'month_trans']].merge(df,
                                        left_on='recordingname',
                                        right_on='recordingname')
journals_merge['wps'] = journals_merge['word_len'] / journals_merge['duration']
# journals_merge.to_csv('../data/all_journals_audio_merged.csv', index=False)<jupyter_output><empty_output><jupyter_text>Putting it altogether<jupyter_code>X = journals_merge[['timeofday', 'month_trans', 'sum_pprn', 'pprn_ratio',
    'wps', 'season', 'median_pitch',
    'max_amp','var_zcr']]
X_dums = pd.get_dummies(X)
y = journals_merge['sentiment']
X = journals_sub[[
    'year', 'day', 'timeofday', 'month_trans', 'sum_pprn', 'pprn_ratio',
    'word_len', 'season', 'inner_frac'
]]
X_dums = pd.get_dummies(X)
y = journals_sub['sentiment_level']
num_feats = X_dums.values

training_data = sparse.hstack((mat, num_feats))
num_feats = X_dums.values
training_data = sparse.hstack((mat, num_feats))<jupyter_output><empty_output><jupyter_text>Prediction Train-Test splits<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(
    training_data,
    y,
    test_size=0.30,
    random_state=42)
reg = sklearn.ensemble.RandomForestRegressor().fit(X_train, y_train)
# reg = sklearn.ensemble.RandomForestClassifier().fit(X_train, y_train)
# plt.figure(figsize=(20, 10))
y_pred = reg.predict(X_test)
# cm = sklearn.metrics.confusion_matrix(y_test, y_pred)
# ax = ConfusionMatrixDisplay(cm, display_labels = y_test.unique()).plot()
# print(sklearn.metrics.accuracy_score(y_test, y_pred))
# print(sklearn.metrics.precision_score(y_test, y_pred, average='weighted'))

plt.figure(figsize=(20, 10))
y_pred = reg.predict(X_test)
print('R2 score: %s' % sklearn.metrics.r2_score(y_test, y_pred))
print('MSE score: %s' % sklearn.metrics.mean_squared_error(y_test, y_pred))
print('%var. exp. score ', sklearn.metrics.explained_variance_score(y_test, y_pred))

y_test
feature_names = vectorizer.get_feature_names() + list(X_dums.columns)
features_x = []
features_y = []
for i in np.where(reg.feature_importances_ > 0.01)[0]:
    features_x += [feature_names[i]]
    features_y += [reg.feature_importances_[i]]
#     print(feature_names[i], reg.feature_importances_[i])
# print(features_sub)
sns.barplot(x=features_y, y=features_x)
plt.xlabel('Feature importance')
# reg = sklearn.ensemble.GradientBoostingClassifier(
#                                 n_estimators=1000, max_depth=10,
#                                 learning_rate=.4, min_samples_leaf=10,
#                                 min_samples_split=9).fit(X_train,y_train)
# y_pred = reg.predict(X_test)
# cm = sklearn.metrics.confusion_matrix(y_test, y_pred)
# ConfusionMatrixDisplay(cm, display_labels=y_test.unique()).plot()
# print(sklearn.metrics.accuracy_score(y_test, y_pred))
# print(sklearn.metrics.precision_score(y_test, y_pred, average='weighted'))
# reg = DecisionTreeClassifier().fit(X_train,y_train)
# y_pred = reg.predict(X_test)
# cm = sklearn.metrics.confusion_matrix(y_test, y_pred)
# ConfusionMatrixDisplay(cm, display_labels=y_test.unique()).plot()
# print(sklearn.metrics.accuracy_score(y_test, y_pred))
# print(sklearn.metrics.precision_score(y_test, y_pred, average='weighted'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15648322808168622, 'fraction_numerical': 0.012630197654391865, 'mean_word_length': 4.352941176470588, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 51, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '428244', 'license': 'MIT', 'n_tokens_mistral': 4515, 'n_tokens_neox': 4234, 'n_words': 661, 'path': 'notebooks/.ipynb_checkpoints/Maslo journal entries data-checkpoint.ipynb', 'repo_name': 'HeyMaslo/AllTheFeels', 'size': '12193'}