<jupyter_start><jupyter_text>Import Ecerything<jupyter_code>import sys
import pandas as pd
import numpy as np
import matplotlib as plt

import sklearn
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from category_encoders import OneHotEncoder, OrdinalEncoder
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline
from sklearn.metrics import accuracy_score

import xgboost
from xgboost import XGBClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier

import pickle
import joblib
from sklearn.linear_model import Ridge, LinearRegression, LogisticRegression
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier, GradientBoostingClassifier
from xgboost import XGBRegressor, XGBClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, accuracy_score, plot_confusion_matrix, plot_roc_curve, classification_report

import sklearn
from sklearn.model_selection import train_test_split
from category_encoders import OneHotEncoder, OrdinalEncoder
!pip install catboost
import catboost
from catboost import CatBoostClassifier
!pip install pdpbox
from pdpbox.pdp import pdp_isolate, pdp_plot, pdp_interact, pdp_interact_plot
!pip install shap
import shap<jupyter_output><empty_output><jupyter_text>Pandas Setting<jupyter_code>pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)<jupyter_output><empty_output><jupyter_text>Combining One Set of CSV from Mid Year Months (June, execept Oct. in 2015)<jupyter_code>df = pd.read_csv("Kickstarter003.csv")
df_0 = pd.read_csv("Kickstarter004.csv")
df_1 = pd.read_csv("Kickstarter006.csv")
df_2 = pd.read_csv("Kickstarter010.csv")
df_3 = pd.read_csv("Kickstarter013.csv")
df_4 = pd.read_csv("Kickstarter015.csv")
df_5 = pd.read_csv("Kickstarter016.csv")
df_main = pd.concat([df, df_0, df_1, df_2, df_3, df_4, df_5])
df_main.shape
df_main
df_main.columns
# permissions, friend, country_displayable_name, usd_exchange_rate, projects, total_hits, seed, search_url, 
# has_more, unread_messages_count, unseen_activity_count, last_update_published_at, converted_pledged_amount  
# is_backing, is_starrable, is_starred, 
df_main.isnull().sum()
df_main = df_main.drop(columns=["permissions", "friends", "country_displayable_name", "usd_exchange_rate", "projects", 
                                "total_hits", "seed", "search_url", 
"has_more", "unread_messages_count", "unseen_activity_count", "last_update_published_at", "converted_pledged_amount",  
"is_backing", "is_starrable", "is_starred"])
df_main.shape
df_main = df_main.dropna()
df_main.shape
# Reading in the Dataset After all the Null Are Dropped
# The Dataset to Work From

df = pd.read_csv("V1.csv")
df.shape
df = df.drop(columns =["Unnamed: 0"])
df.shape<jupyter_output><empty_output><jupyter_text>Day 1-- Data Cleaning Goal1. Extract Subcategory and Category2. Drop live and cancelled from state column and only leave success and failed3. Get the campaign duration<jupyter_code>df.info()
# Looks Like most of the missing values are on the same row
# Let's Just Drop Them All-- 
# The crosssection time series of mid year data for 5 years (June, except Oct. in 2015) is big enough anyways

df = df.dropna()
df.shape
# The Shape is the same
df.isnull().sum()
df[df.blurb.isnull() == True]
df['blurb_length'] = df['blurb'].str.split().str.len()
df['goal_in_usd'] = round(df['goal'] * df["static_usd_rate"],2)
df['blurb_length']
# Get the Campaign Duration 
#Convert epoch dates to Date using pandas
df['created_at'] = pd.to_datetime(df['created_at'], origin='unix', unit='s').dt.date
df['deadline'] = pd.to_datetime(df['deadline'], origin='unix', unit='s').dt.date
df['launched_at'] = pd.to_datetime(df['launched_at'], origin='unix', unit='s').dt.date
df['state_changed_at'] = pd.to_datetime(df['state_changed_at'], origin='unix', unit='s').dt.date
# Converting into datetime type
df['created_at'] = pd.to_datetime(df['created_at'],format='%Y-%m-%d')
df['deadline'] = pd.to_datetime(df['deadline'],format='%Y-%m-%d')
df['launched_at'] = pd.to_datetime(df['launched_at'],format='%Y-%m-%d')

# New feature - campaign duration
df['campaign_duration'] = (df['deadline'] - df['launched_at']).dt.days
df['campaign_duration'][:5]
def Custom_Parser(data):
    if data is not np.NaN:
        try:
            j1 = json.loads(data)
        except: 
            return data.split(":",1)[1].replace("'", "")
# !pip install json
# import json
# df["w/e"] = df["category"].apply(Custom_Parser)
df["w/e"]
# Example category value
df.iloc[190]['category']
fun_subcategory = lambda x: x['category'].split('/')[1].split('","position')[0]
df['sub_category'] = df.apply(fun_subcategory, axis=1)
df['sub_category'].head()
# Extracting category section and replacing the original category variable
fun_category = lambda x: x['category'].split('"slug":"')[1].split('/')[0]
df['category'] = df.apply(fun_category, axis=1)
df['category'].head()
df['state'].value_counts()
# droppping live status
condition = df['state'] == 'live'
df = df[~condition]

condition_1 = df['state'] == 'canceled'
df = df[~condition_1]


condition_2 = df['state'] == 'suspended'
df = df[~condition_2]
df['state'].value_counts()
# replacing cancelled with failed
df.head(2)
df_1 = df[["backers_count", "category", "campaign_duration", 
    "goal_in_usd", "blurb_length", 
    "sub_category", "state", "usd_pledged"]]
df_2 = df_1.to_csv("V2.csv")
df_2 = pd.read_csv("V2.csv")
df_2["state"]
df_2.loc[df_2['state']=='successful','state'] = 1
df_2.loc[df_2['state']=='failed','state'] = 0
df_2['state'] = df_2['state'].astype(int)
df_2['state'].value_counts()
df_2.info()
df_3 = df_2.to_csv("V3.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13604211218950485, 'fraction_numerical': 0.01612107254482645, 'mean_word_length': 6.855297157622739, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '261081', 'license': 'MIT', 'n_tokens_mistral': 2175, 'n_tokens_neox': 2095, 'n_words': 470, 'path': 'Notebooks/Data Cleaning and Processing.ipynb', 'repo_name': 'pflee1989/kickstarter2', 'size': '6079'}