<jupyter_start><jupyter_text>Empty Textremove [delete] / [removed] / [deleted] text<jupyter_code>#remove unwated text in self_text
df = df[~df['self_text'].str.contains('[removed]',
                                     regex=False)]
#removed unwanted text in author
df = df[~df['author'].str.contains('[deleted]',
                                     regex=False)]

#removed unwanted text in title
df = df[~df['title'].str.contains('[deleted]',
                                     regex=False)]<jupyter_output><empty_output><jupyter_text>Drop DuplicatesAssume that it's impossible for two posts to be identical AND created at the same time by the same person, and somehow get to top/hot page.<jupyter_code>#drop duplicates
df_dup = df.drop_duplicates(subset = ['author', 'created_utc'],
                            keep='first', 
                            ignore_index=False)

df_dup.shape
len(df_dup) / len(df)
df_dup.head()<jupyter_output><empty_output><jupyter_text>Organize By  Submission Date & Fill In Num SubscribersMake this assumption:The number of subscribers at the time of your post is whatever the closest known number is<jupyter_code>def sortFillDates(df, col=None):
    """
    returns a copy of df where it is sorted by column
    """
    if not col:
        return df
    
    df_date = df.copy(deep=True)
    
    df_date.sort_values(by=col,
                        inplace=True
                       )
    
    df_date.interpolate(method='linear', 
                        columns=[col], 
                        inplace=True)
    
    df_date = df_date.reset_index(drop=True)
    
    
    return df_date

    

    
df_date = sortFillDates(df_dup, 'created_utc')
#change to readable friendly time
df_date['created_utc'] = pd.to_datetime(df_date['created_utc'], unit='s')

df_date.head()
df_date['subreddit_subscribers'].isna().mean()<jupyter_output><empty_output><jupyter_text>What about posts that stay popular over timeframe?  Get Time Differential<jupyter_code>#get first mention of comment
df_first = df_date.drop_duplicates(subset=['self_text', 'title'], keep='first').reset_index(drop=True)
#get last mention of comment
df_last = df_date.drop_duplicates(subset=['self_text', 'title'], keep='last').reset_index(drop=True)

#measure time differnece
created_time_diff =  (df_last['created_utc'] - df_first['created_utc'])

#make into float hours
created_time_diff_hr = created_time_diff/ pd.to_timedelta(1, unit='D')
created_time_diff_hr.hist()
df_date.columns
df_date.hist(column='total_awards_received', bins=np.arange(0, 20, 1))<jupyter_output><empty_output><jupyter_text>put it in a function<jupyter_code>def sigmoid_modified(x):
    return 0.5 + (1 / (1 + math.exp(-x)))

def timeDiffDuplicates(df, cols, time_col):
    
    #get first mention of comment
    df_first = df.drop_duplicates(subset=cols, keep='first').reset_index(drop=True)
    #get last mention of comment
    df_last = df.drop_duplicates(subset=cols, keep='last').reset_index(drop=True)

    #measure time differnece
    created_time_diff =  (df_last[time_col] - df_first[time_col])

    #make into float hours
    created_time_diff_hr = created_time_diff/ pd.to_timedelta(1, unit='D')
    
#     #apply sigmoid function
#     time_amplification = created_time_diff_hr.apply(sigmoid_modified)
    
    #make deep copy
    df_time_diff = df.copy(deep=True)
    
    #drop duplicates
    df_time_diff = df.drop_duplicates(subset=cols, keep='first').reset_index(drop=True)
    
    #add column
    df_time_diff['created_time_diff_hr'] = created_time_diff_hr
    
    return df_time_diff
    
df_time_diff = timeDiffDuplicates(df = df_date,
                                  cols=['title', 'self_text'],
                                  time_col='created_utc')
df_time_diff.shape
df_time_diff.hist('created_time_diff_hr')
len(df_time_diff) / len(df_date)<jupyter_output><empty_output><jupyter_text>NaN Values  Flair<jupyter_code>#fill NaN with 'no_flair'
df_time_diff['flair'] = df_time_diff['flair'].fillna('no_flair')
df_time_diff['flair'].value_counts(ascending=False)
def seriesTopK(series, k=10):
    """
    returns the top k values
    """
    return series.value_counts(ascending=False)[:k].index
def fixFlair(df, k=10):
    """
    returns a new dataframe with dummies on flair column
    """
    
    #get the top k values by count
    flair_top_vals = seriesTopK(df['flair'], k=10)
    
    #make a deep copy
    df_flair = df.copy(deep=True)
    
    #keep only rows where flair is in the top k
    df_flair = df_flair[df_flair['flair'].isin(flair_top_vals)]
    
    #get the dummies
    df_flair = pd.get_dummies(df_flair, #the dataframe
                              prefix='flair', # prefix to be added
                              columns=['flair'] #columns to be dummied
                             )
    
    
    return df_flair


df_flair = fixFlair(df_time_diff, k=10)
df_flair.shape<jupyter_output><empty_output><jupyter_text>Collect Emojis<jupyter_code>def collectEmojisFromString(text):
    """
    returns a dictionary count of all emojis in a descriptive manner
    """
    
    #get location of all emojis
    emoj_list_loc = emoji.emoji_lis(text)
    
    #make a dictionary
    emoj_dict = {}
    
    for d in emoj_list_loc:
        #de-emojify
        demoj = emoji.demojize(d['emoji'])
        if demoj in emoj_dict:
            emoj_dict[demoj] += 1
        else:
            emoj_dict[demoj] = 1
    
    return emoj_dict
        
    
emoj_list = []

#get the emojis from the two text columns
for self_text, title in zip(df_flair['self_text'], df_flair['title']):
    
    #combine 
    text = ' '.join([self_text, title])
    
    #make a dicto f emojis
    emoj_dict = collectEmojisFromString(text)
    
    #append to list
    emoj_list.append(emoj_dict)
    
#make a dataframe
df_emoj =pd.DataFrame(emoj_list)

#get sum by columns
emoji_col_sum = df_emoj.sum(axis=0)
#keep top 10 emojis
df_emoj_count = df_emoj[emoji_col_sum.sort_values(ascending=False).index[:10]]

#fill missing vals by 0
df_emoj_count.fillna(value=0, inplace=True)
df_emoj_count.shape
#merge
df_emoj = df_flair.merge(right=df_emoj_count,
                         how='left',
                        left_on=df_flair.index, 
                        right_on=df_emoj_count.index,
                         right_index=True,
                         left_index=True)


#fill empty emojis
df_emoj[df_emoj_count.columns] = df_emoj[df_emoj_count.columns].fillna(value = 0)


#check shape
df_emoj.shape
df_emoj.head()
#final check for missing values
df_emoj.isna().mean()<jupyter_output><empty_output><jupyter_text>Create Amplification ScoreConsider a list below a measure popularity among popular posts. This does give advantage to submissions over comments, and I believe this is valid since submission titles are the first thing we see before diving into a thread.- total_awards_received- num_comments- upvote_ratio<jupyter_code>df_emoj.columns
df_emoj[['total_awards_received', 'num_comments', 'upvote_ratio','created_time_diff_hr']].describe()
df_emoj.hist(column='total_awards_received', bins=np.arange(0, 100, 1))
df_emoj.hist(column='num_comments', bins=np.arange(0, 20000, 1000))
df_emoj.hist(column='upvote_ratio')
#let's create a copy
df_amp = df_emoj.copy(deep=True)

#create amplification column
df_amp['amplification'] = ((df_amp['created_time_diff_hr'] +1) 
                           * (df_amp['upvote_ratio'] +1 ) 
                           * (1+ np.log10(1 + df_amp['num_comments'])) 
                           * (1+ np.log10(1 + df_amp['total_awards_received'])))<jupyter_output><empty_output><jupyter_text>Save the Work<jupyter_code>df_amp.to_csv(os.path.join('data', 'data_munged.csv'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1082687338501292, 'fraction_numerical': 0.0068475452196382425, 'mean_word_length': 3.2532967032967033, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '249264', 'license': 'MIT', 'n_tokens_mistral': 2724, 'n_tokens_neox': 2636, 'n_words': 534, 'path': 'combine_munge.ipynb', 'repo_name': 'NadimKawwa/WSB_analysis', 'size': '7740'}