<jupyter_start><jupyter_text>KLSE - Preprocess, Cluster/Categorize the Stocks<jupyter_code># To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import pandas as pd
import datetime as dt
import calendar as cal
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "end_to_end_project"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)

def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)

# Ignore useless warnings (see SciPy issue #5998)
import warnings
warnings.filterwarnings(action="ignore", module="scipy", message="^internal gelsd")<jupyter_output><empty_output><jupyter_text>Load the stock master file<jupyter_code>df_stocks = pd.read_csv('../dataset/KLSE_investing.csv', index_col=['name'])
df_stocks.head(10)<jupyter_output><empty_output><jupyter_text>Load dividends file<jupyter_code>df_dividends = pd.read_csv('../dataset/KLSE_dividends_investing.csv')
df_dividends.info()
df_dividends.describe()
df_dividends.columns<jupyter_output><empty_output><jupyter_text>Display some data<jupyter_code>df_dividends.head(10)<jupyter_output><empty_output><jupyter_text>Preprocess the data for training<jupyter_code>df_dividends_train = df_dividends.copy()
df_dividends_train['dividend_year'] = pd.to_datetime(df_dividends_train['ex_dividend_date'], format='%b %d, %Y').dt.year
#df_dividends_train = df_dividends.drop(['ex_dividend_date', 'payment_date'], axis=1)
# df_dividends_train['yield'] = df_dividends_train['yield'].map({'-': '0'})
df_dividends_train['yield'] = df_dividends_train['yield'].replace('-','0')
df_dividends_train['yield'] = df_dividends_train['yield'].str.rstrip('%').astype('float')
df_dividends_train.head()<jupyter_output><empty_output><jupyter_text>Apply one-hot encoding to code<jupyter_code>from sklearn.preprocessing import LabelEncoder, OneHotEncoder

label_encoder = LabelEncoder()
code = df_dividends_train['code']
code_encoded = label_encoder.fit_transform(code)
code_encoded
print(label_encoder.classes_)
one_hot_encoder = OneHotEncoder()
code_1hot = one_hot_encoder.fit_transform(code_encoded.reshape(-1,1))
code_1hot
code_1hot.toarray()<jupyter_output><empty_output><jupyter_text>Use LabelBinarizer<jupyter_code>from sklearn.preprocessing import LabelBinarizer
label_binarizer = LabelBinarizer()
code_1hot = label_binarizer.fit_transform(code)
code_1hot<jupyter_output><empty_output><jupyter_text>Build the preprocessing pipelines  Custom Data Processor<jupyter_code>from sklearn.base import BaseEstimator, TransformerMixin
from datetime import datetime
import numpy.core.defchararray as npf
df_dividends = pd.read_csv('../dataset/KLSE_dividends_investing.csv')
df_dividends.info()
df_dividends.head(10)
# Create a class to select numerical or categorical columns 
# since Scikit-Learn doesn't handle DataFrames yet
class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names].values

code_index, name_index, ex_dividend_date_index, dividend_index, payment_date_index, yield_index = 0, 1, 2, 3, 4, 5 

class KlseDataPreprocessor(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        return self  # nothing else to do
    def transform(self, X, y=None):
        
        ## Create dividend_year attribute
        # print(X[:, ex_dividend_date_index])
        dividend_year = pd.to_datetime(X[:, ex_dividend_date_index], format='%b %d, %Y').year
        # dividend_year = np.array(dividend_year, dtype=np.integer)
        # print(dividend_year)
        
        ## Convert yield to float
        # yields = X[:, yield_index]
        # print(yields)
        # print(yields[yields == '-'])
        X[:, yield_index][X[:, yield_index] == '-'] = '0'
        # print(yields[yields == '0'])
        X[:, yield_index] = npf.rstrip(np.array(X[:, yield_index], dtype='str'), '%').astype(np.float)
        # print(yields)
        
        ## Drop string fields
        X = np.delete(X, [code_index, name_index, ex_dividend_date_index, payment_date_index], axis=1)
        
        result = np.c_[X, dividend_year]
        print(result)
        return result
    
        # X['dividend_year'] = pd.to_datetime(X['ex_dividend_date'], format='%b %d, %Y').dt.year
        # X['yield'] = X['yield'].replace('-','0')
        # X['yield'] = X['yield'].str.rstrip('%').astype('float')
        # return X.drop(['ex_dividend_date','payment_date', 'code', 'name'], axis=1)   # Drop non numerical columns
df_dividends = pd.read_csv('../dataset/KLSE_dividends_investing.csv')
klse_data_preprocessor = KlseDataPreprocessor()
klse_data_preprocessor.transform(df_dividends.values)<jupyter_output>[[0.01 2.94 2017]
 [0.0135 0.72 2015]
 [0.017 1.3 2018]
 ...
 [0.0095 2.84 2018]
 [0.049 5.1 2018]
 [0.06 6.12 2018]]<jupyter_text>Check scikit version<jupyter_code>print(list(df_dividends))
import nltk
import sklearn

print('The nltk version is {}.'.format(nltk.__version__))
print('The scikit-learn version is {}.'.format(sklearn.__version__))<jupyter_output>['code', 'name', 'ex_dividend_date', 'dividend', 'payment_date', 'yield']
The nltk version is 3.2.5.
The scikit-learn version is 0.19.2.<jupyter_text>Build the pipelines<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from future_encoders import OneHotEncoder

df_dividends = pd.read_csv('../dataset/KLSE_dividends_investing.csv')
df_dividends['code_id'] = df_dividends.groupby('code').ngroup()
#df_dividends.groupby(['code_id','year']).agg({'amount': np.sum})

standard_attribs = list(df_dividends)
code_attrib = ["code"]

#print(standard_attribs)
# print(df_dividends)

standard_pipeline = Pipeline([
        ('selector', DataFrameSelector(standard_attribs)),
        ('preprocessor', KlseDataPreprocessor()),
        ('scaler', StandardScaler()),
    ])

code_pipeline = Pipeline([
        ('selector', DataFrameSelector(code_attrib)),
        ('code_encoder', OneHotEncoder(sparse=False))
    ])

from sklearn.pipeline import FeatureUnion

full_pipeline = FeatureUnion(transformer_list=[
        ("standard_attribs", standard_pipeline)
    ])

df_prepared = full_pipeline.fit_transform(df_dividends)
df_prepared

df_dividends.head(10)
df_prepared.shape<jupyter_output><empty_output><jupyter_text>Categorize the stocks<jupyter_code>from sklearn.cluster import KMeans
from adspy_shared_utilities import plot_labelled_scatter

kmeans = KMeans(n_clusters = 3)
kmeans.fit(df_prepared)
print(kmeans.inertia_)

plot_labelled_scatter(df_prepared, kmeans.labels_, 
                      ['Cluster 1', 'Cluster 2', 'Cluster 3'])<jupyter_output>6123.037889465623
{'fraction_non_alphanumeric': 0.12733215307095191, 'fraction_numerical': 0.018657224567615417, 'mean_word_length': 5.60431654676259, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '630817', 'license': 'MIT', 'n_tokens_mistral': 2704, 'n_tokens_neox': 2562, 'n_words': 461, 'path': 'notebook/KLSE_test_1.ipynb', 'repo_name': 'mengwangk/myinvestor-toolkit', 'size': '7343'}