<jupyter_start><jupyter_text>Assumed steps Benchmark    - dataset_v0         EVALUATE         - [catboost_v0] untuned classifier on dataset_v0 data set (~75% accuracy)        - [catboost_v1] tuned classifier on dataset_v0 data set          Data set Preprocessing    - dataset_v1        - data cleaning            - ignore the tuples             - missing values                - fill with mean | meadian | custom            - elinimate noise data                 - binning method & regression & clustering         - data transformation            - normalization            - attribute selection            - discretization  muted        - data reduction  muted         CAVEATS - Imbalanced target (700 vs 300)  resolved within shuffling  could be applied sampling as well- Odd mean loan of creditors age of > 65   Steps1. Rename columns names for explicit visual understanding of features<jupyter_code>class Transformer:
    @staticmethod
    def rename_columns(dataset: pd.DataFrame):
        """Rename dataframe columns names.
        Notes:
            target column names based on provided data_description.txt
        Returns:
            pd.DataFrame - dataframe with renamed columns 
        """
    
        target_columns = {
            'X01': 'account_status',
            'X06': 'account_savings',
            'X02': 'credit_duration',
            'X03': 'credit_history',
            'X04': 'credit_purpose',
            'X05': 'credit_amount',
            'X07': 'employment_status',
            'X17': 'employment_description',
            'X08': 'income_installment_rate',
            'X09': 'gender_status',
            'X10': 'credit_guarantors',
            'X11': 'residence',
            'X12': 'owned_property',
            'X13': 'age',
            'X14': 'installment_plans',
            'X15': 'accomondation_type',
            'X16': 'credit_existing_number',
            'X18': 'liable_maintain',
            'X19': 'phone_number',
            'X20': 'foreign_worker',
            'Y': 'y'
        }
        return dataset.rename(columns=target_columns)    
    
    @staticmethod
    def categorical_columns(dataset: pd.DataFrame):
        """Within dataset find categorical columns
        
        Returns:
          List: list of names of categorical columns.
        """
        categorical_columns_indexes = []
        
        columns = dataset.columns
        num_columns = dataset._get_numeric_data().columns # get numerical columns
        
        categorical_columns = sorted(list(set(columns) - set(num_columns)))
        categorical_columns_indexes = [dataset.columns.get_loc(column) for column in categorical_columns]
        return categorical_columns, categorical_columns_indexes
          
    @staticmethod
    def train_test_split(dataset: pd.DataFrame, test_size: int, shuffle=False):
        from sklearn.model_selection import train_test_split
        print(f'Splitting dataset {dataset.shape} with test_size :{test_size}')
        X = dataset.drop(columns=['y'])
        y = dataset['y']
        X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=shuffle, test_size=test_size)
        print(f'X_train: {X_train.shape}')
        print(f'y_train: {y_train.shape}')
        print(f'X_test: {X_test.shape}')
        print(f'y_test: {y_test.shape}')
        return X_train, X_test, y_train, y_test
    
    @staticmethod
    def custom_split(dataset: pd.DataFrame, test_percentage: int):
        size = dataset.shape[0]
        train_ratio = (size * test_percentage) // 100
        test_ratio = size - train_ratio

        X_train = pd.DataFrame(dataset[:train_ratio, :-1])
        X_test = pd.DataFrame(dataset[:test_ratio, :-1])

        y_train = dataset[:train_ratio, -1]
        y_test = dataset[:test_ratio, -1]
        return X_train, X_test, y_train, y_test
    
    @staticmethod
    def data_cleaning(dataframe: pd.DataFrame):
        """Applies follwoing transformations:   
            - data cleaning
                - ignore the tuples # when the dataset is large and multiple values are missing within a tuple.
                - missing values
                    - fill with mean | meadian | custom
                - encoding
        """
        categorical_columns = [column for column in dataset_v1.columns if dataset_v1[f'{column}'].dtype == 'object']
    
        for column_name in categorical_columns: 
            labels = dataframe[f'{column_name}'].astype('category').cat.categories.tolist()
            replace_map_comp = {f'{column_name}' : {k: v for k,v in zip(labels,list(range(1,len(labels)+1)))}}
            dataframe.replace(replace_map_comp, inplace=True)
        return dataframe
    
    @staticmethod
    def data_transformation(dataframe: pd.DataFrame, features=20):
        """Applies follwoing transformations:
            - data transformation
                - normalization # preprocessing.MinMaxScaler()
                - attribute selection # SelectKBest(chi2, k=features)
                - discretization # muted for now 
        """
        from sklearn import preprocessing
        from sklearn.feature_selection import chi2
        from sklearn.feature_selection import SelectKBest
        original_columns = ['account_status', 'credit_duration', 'credit_history', 'credit_purpose', 'credit_amount', 'account_savings',
                            'employment_status', 'income_installment_rate', 'gender_status','credit_guarantors', 'residence', 'owned_property', 'age',
                            'installment_plans', 'accomondation_type', 'credit_existing_number', 'employment_description',
                            'liable_maintain', 'phone_number', 'foreign_worker', 'y'
                           ]
        
        values = dataframe.values #returns a numpy array
        min_max_scaler = preprocessing.MinMaxScaler()
        x_scaled = min_max_scaler.fit_transform(values)
        buff_df = pd.DataFrame(x_scaled)
        
        buff_df.columns = original_columns # cast back original column namings 
        
        X = buff_df.drop(columns=['y'], axis=1)
        y = buff_df['y']
        
        dataframe = SelectKBest(chi2, k=features).fit_transform(X, y)
        return dataframe
    
class Models:
    @staticmethod
    def _catboost_classifier(X_train, X_test, y_train, y_test, cat_features, hparams=False):
        """CatBoost Classifier.
        
        Notes:
          - cat_features: CatBoost model requires list of indexes which denote categorical columns;
        """
        from catboost import CatBoostClassifier, Pool
        from sklearn.metrics import accuracy_score
        from sklearn.model_selection import GridSearchCV
        
        if hparams == False:
            train_data = Pool(
                data=X_train.values,
                label=y_train,
                cat_features=cat_features
            )
            model = CatBoostClassifier(iterations=30)
            model.fit(train_data)
            
            predictions = model.predict(X_test.values)
            accuracy = accuracy_score(y_test, predictions)
            return accuracy
        
        else:
            model = CatBoostClassifier()
            
            grid = {
                'learning_rate': [0.03, 0.1],
                'depth': [4, 6, 10],
                'l2_leaf_reg': [1, 3, 5, 7, 9]
            }
            grid = GridSearchCV(estimator=model,
                                param_grid=grid,
                                cv=3,
                                n_jobs=-1
                               )
            grid.fit(X_train, y_train)
            predictions = grid.best_estimator_.predict(X_test.values)
            accuracy = accuracy_score(y_test, predictions)
            return accuracy
                    
        
    @staticmethod
    def _xgboost_classifier(X_train, X_test, y_train, y_test, hparams=False):
        """Apply XGBoost Classifier.
        Notes:
          When using XGBoost we need to convert categorical variables into numeric.
        """
        from xgboost import XGBClassifier
        from sklearn.metrics import accuracy_score
        # untuned model
        if hparams==False:
            model = XGBClassifier()
            model.fit(X_train, y_train, verbose=True)
            predictions = model.predict(X_test)
            accuracy = accuracy_score(y_test, predictions)
        
        # tuned model
        else:
            model = XGBClassifier(silent=False, 
                                  scale_pos_weight=1,
                                  learning_rate=0.01,  
                                  subsample = 0.8,
                                  objective='binary:logistic', 
                                  n_estimators=100, 
                                  reg_alpha = 0.3,
                                  max_depth=2,
                                  seed=42
                                 )
            
            model.fit(X_train, y_train, verbose=True)
            predictions = model.predict(X_test)
            accuracy = accuracy_score(y_test, predictions)
            
        return accuracy<jupyter_output><empty_output><jupyter_text>Entry Point<jupyter_code>models = Models()
transformer = Transformer()


row_data = pd.read_csv('./dataset/project_data.csv', delimiter=';')
# print(row_data)


# # Benchmark
dataset_v0 = transformer.rename_columns(row_data)
categorical_columns, categorical_columns_indexes = transformer.categorical_columns(dataset_v0) 
X_train_v0, X_test_v0, y_train_v0, y_test_v0 = transformer.train_test_split(dataset_v0,
                                                                            test_size=.3,
                                                                            shuffle=True
                                                                           )

# - dataset_v0 && - [catboost_v0] untuned classifier on dataset_v0 data set (~75% accuracy)
dataset_v0_catboost_v0 = models._catboost_classifier(X_train_v0,
                                                     X_test_v0,
                                                     y_train_v0,
                                                     y_test_v0,
                                                     cat_features=categorical_columns_indexes,
                                                     hparams=False
                                                    )


# Beating Benchmark
dataset_v1 = transformer.rename_columns(row_data)
dataset_v1 = transformer.data_cleaning(dataset_v1)
dataset_v1 = transformer.data_transformation(dataset_v1, features=5) # return ndarray
X_train_v1, X_test_v1, y_train_v1, y_test_v1 = transformer.custom_split(dataset_v1,
                                                                        test_percentage=30
                                                                       )


# - dataset_v1 && - [catboost_v0] untuned classifier 
dataset_v1_catboost_v0 = models._catboost_classifier(X_train_v1,
                                                     X_test_v1,
                                                     y_train_v1,
                                                     y_test_v1,
                                                     cat_features=[],
                                                     hparams=False
                                                    )

# - dataset_v1 && - [xgboost_classifier_v0] untuned classifier
dataset_v1_xgboost_classifier_v0 = models._xgboost_classifier(X_train_v1,
                                                              X_test_v1,
                                                              y_train_v1,
                                                              y_test_v1,
                                                              hparams=False
                                                             )

# - dataset_v1 && - [catboost_v1] tuned classifier 
dataset_v1_catboost_v1 = models._catboost_classifier(X_train_v1,
                                                     X_test_v1,
                                                     y_train_v1,
                                                     y_test_v1,
                                                     cat_features=[],
                                                     hparams=True
                                                    )

# - dataset_v1 && - [xgboost_classifier_v1] tuned classifier
dataset_v1_xgboost_classifier_v1 = models._xgboost_classifier(X_train_v1,
                                                              X_test_v1,
                                                              y_train_v1,
                                                              y_test_v1,
                                                              hparams=True
                                                             )

# model_version > 0, denones tuned model

results = [[dataset_v0_catboost_v0,
           dataset_v1_catboost_v0, 
           dataset_v1_catboost_v1, # tuned
           dataset_v1_xgboost_classifier_v0,
           dataset_v1_xgboost_classifier_v1 # tuned
          ]] 

result_df = pd.DataFrame(results,
                         columns=['dataset_v0_catboost_v0',
                                  'dataset_v1_catboost_v0',
                                  'dataset_v1_catboost_v1',
                                  'dataset_v1_xgboost_classifier_v0',
                                  'dataset_v1_xgboost_classifier_v1'],
                        index=['accuracy'])
result_df<jupyter_output><empty_output><jupyter_text>Cluster customers into various groups<jupyter_code>import matplotlib.pyplot as plt
from kneed import KneeLocator
from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.preprocessing import StandardScaler
kmeans = KMeans(
    init="random",
    n_clusters=3,
    n_init=10,
    max_iter=300,
    random_state=42
)
kmeans.fit(X_train_v1)
print('The lowest SSE value:', kmeans.inertia_)
print('Final locations of the centroid:', kmeans.cluster_centers_)
print('The number of iterations required to converge:', kmeans.n_iter_)
print(kmeans.labels_)
kmeans_kwargs = {
    "init": "random",
    "n_init": 10,
    "max_iter": 300,
    "random_state": 42,
}

# A list holds the SSE values for each k
sse = []
for k in range(1, 11):
    kmeans = KMeans(n_clusters=k, **kmeans_kwargs)
    kmeans.fit(X_train_v1)
    sse.append(kmeans.inertia_)
 plt.style.use("fivethirtyeight")
 plt.plot(range(1, 11), sse)
 plt.xticks(range(1, 11))
 plt.xlabel("Number of Clusters")
 plt.ylabel("SSE")
 plt.show()
kl = KneeLocator(
    range(1, 11), sse, curve="convex", direction="decreasing")

kl.elbow
# A list holds the silhouette coefficients for each k
silhouette_coefficients = []

# Notice you start at 2 clusters for silhouette coefficient
for k in range(2, 11):
    kmeans = KMeans(n_clusters=k, **kmeans_kwargs)
    kmeans.fit(X_train_v1)
    score = silhouette_score(X_train_v1, kmeans.labels_)
    silhouette_coefficients.append(score)
plt.style.use("fivethirtyeight")
plt.plot(range(2, 11), silhouette_coefficients)
plt.xticks(range(2, 11))
plt.xlabel("Number of Clusters")
plt.ylabel("Silhouette Coefficient")
plt.show()
from sklearn.cluster import DBSCAN
from sklearn.datasets import make_moons
from sklearn.metrics import adjusted_rand_score
# Instantiate k-means and dbscan algorithms
kmeans = KMeans(n_clusters=2)
dbscan = DBSCAN(eps=0.3)

# Fit the algorithms to the features
kmeans.fit(X_train_v1)
dbscan.fit(X_train_v1)

# Compute the silhouette scores for each algorithm
kmeans_silhouette = silhouette_score(
    X_train_v1, kmeans.labels_
).round(2)
dbscan_silhouette = silhouette_score(
   X_train_v1, dbscan.labels_
).round (2)
print(kmeans_silhouette, dbscan_silhouette)<jupyter_output>0.35 0.06
{'fraction_non_alphanumeric': 0.0788494291301331, 'fraction_numerical': 0.015706806282722512, 'mean_word_length': 1.6502841858910064, 'pattern_counts': {'":': 4, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 83, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '417914', 'license': 'MIT', 'n_tokens_mistral': 4482, 'n_tokens_neox': 4239, 'n_words': 880, 'path': 'main.ipynb', 'repo_name': 'KumundzhievMaxim/IntroductionDataScience', 'size': '15853'}