<jupyter_start><jupyter_text>Extended data frame<jupyter_code>df_new['duration'] = (df_new.deadline-df_new.launched_at)/(3600*24)
df_new['duration'] = df_new['duration'].round(2)
df_new['goal_usd'] = df_new['goal'] * df_new['static_usd_rate']
df_new['goal_usd'] = df_new['goal_usd'].round(2)
#df_new['launched_at_full'] = pd.to_datetime(df_new['launched_at'], unit='s')
df_new['launched_at_full'] = pd.to_datetime(df_new['launched_at'], unit='s')
df_new['launched_at_year'] = pd.DatetimeIndex(df_new['launched_at_full']).year
df_new['launched_at_month'] = pd.DatetimeIndex(df_new['launched_at_full']).month
df_new['created_at_full'] = pd.to_datetime(df_new['created_at'], unit='s')
df_new['created_at_year'] = pd.DatetimeIndex(df_new['created_at_full']).year
df_new['created_at_month'] = pd.DatetimeIndex(df_new['created_at_full']).month
df_new['deadline_full'] = pd.to_datetime(df_new['deadline'], unit='s')
df_new['deadline_year'] = pd.DatetimeIndex(df_new['deadline_full']).year
df_new['deadline_month'] = pd.DatetimeIndex(df_new['deadline_full']).month
from math import isnan
category_dict = pd.Series(df_new['category_name'].values,index=df_new['category_id']).to_dict()
def parent_cat_mapper(row):
    if isnan(row['category_parent_id']):
        return row['category_name']
    else:
        return category_dict[row['category_parent_id']]
category_parent_name = df_new.apply(parent_cat_mapper, axis=1)
df_new['category_parent_name'] = category_parent_name
df_new.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 209222 entries, 0 to 209221
Columns: 108 entries, backers_count to category_parent_name
dtypes: bool(5), datetime64[ns](3), float64(18), int64(16), object(66)
memory usage: 165.4+ MB<jupyter_text>Save frame<jupyter_code>save_dataframe(df_new, './data_frame_full_2021-03-12.pickle')<jupyter_output><empty_output><jupyter_text>Reduced data frame<jupyter_code>for i , val in df_new.iloc[60060,:].items():
    print(i)
    print(val)
    print()
survival_lst = ['backers_count', 'blurb', 'country', 'created_at', 'currency', 'deadline','disable_communication', 'goal', 'launched_at','name', 'staff_pick','state', 
                'usd_pledged','usd_type','category_id','category_name','category_slug','category_parent_id', 'category_parent_name', 'location_id', 'location_name','location_type', 
                'photo_key', 'photo_full', 'duration', 'goal_usd', 
                'launched_at_full', 'launched_at_year', 'launched_at_month', 'created_at_full', 'created_at_year', 'created_at_month', 'deadline_full', 'deadline_year', 'deadline_month']
df_eda = df_new[survival_lst]
save_dataframe(df_eda, './data_frame_small_2021-03-12.pickle')
df_eda.head(2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16574990832416575, 'fraction_numerical': 0.023835716905023837, 'mean_word_length': 8.956204379562044, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '667666', 'license': 'MIT', 'n_tokens_mistral': 1078, 'n_tokens_neox': 1023, 'n_words': 122, 'path': 'kickstarter_02_preparation.ipynb', 'repo_name': 'dominikmn/ds-kickstarter-project', 'size': '2727'}