<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Libraries and functions1.1&nbsp;&nbsp;Import libraries1.2&nbsp;&nbsp;Definition of  functions used locally2&nbsp;&nbsp;Options3&nbsp;&nbsp;Load data4&nbsp;&nbsp;Create base indices4.1&nbsp;&nbsp;Market cap weighted4.2&nbsp;&nbsp;Equal weights4.3&nbsp;&nbsp;compute index returns5&nbsp;&nbsp;Index based on model predictions5.1&nbsp;&nbsp;Weight generation5.2&nbsp;&nbsp;Build your strategy5.3&nbsp;&nbsp;Build index5.3.1&nbsp;&nbsp;Monthly index levels  Libraries and functions Import librariesNeed to be able to access functions in base_dir/src<jupyter_code># libraries

# general
import sys
import os

import itertools

import dateutil.relativedelta as relativedelta

import datetime
import numpy as np
import pandas as pd

# plotting
%matplotlib inline

import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn as sns

# # statistics and machine learning

from IPython.display import display
pd.options.display.max_columns = None

# # add the base path to python system path
path = os.getcwd()
#dir_up = os.path.abspath(os.path.join(path, os.pardir))
base_path = os.path.abspath(os.path.join(path, os.pardir))
sys.path.append(base_path)


# from mpl_toolkits.axes_grid.anchored_artists import AnchoredText
from matplotlib import gridspec


# # libraries within package
from src.finance_functions import multiple_returns_from_levels_vec, project_to_first
from src.finance_functions import df_restrict_dates
from src.automotive_dictionaries import equity_name2first_date

from src.index_functionality import index_levels_from_returns

from src.financial_metrics import extract_performance

%load_ext autoreload
%autoreload 2
%reload_ext autoreload<jupyter_output><empty_output><jupyter_text>Options  Load data<jupyter_code>filename = '../data/data_sample_monthly.csv'
infile = filename
df_comb_long = pd.read_csv(infile)
df_comb_long['date'] = pd.to_datetime(df_comb_long['date'])
#df_comb_long.head()
#eq_name = 'Equity Parent'
eq_name = 'company'
df_prices = df_comb_long.pivot(values='stock_price', index='date', columns=eq_name)


# monthly returns
df_returns = multiple_returns_from_levels_vec(df_prices.ffill())<jupyter_output><empty_output><jupyter_text>Create base indices<jupyter_code>df_market_cap = df_comb_long.pivot(values='MarketCap_Mlns', index='date', columns=eq_name)
df_market_cap.index = df_market_cap.index.map(project_to_first)

# deal with the missing values by taking the previously available one
df_market_cap.ffill(inplace=True)

# set to zero when not available, this takes care of the market cap weights

for col in df_market_cap.columns:
    first_date = project_to_first(equity_name2first_date[col])
    mask = df_market_cap.index < first_date
    df_market_cap.loc[mask, col] = 0.0
    #print(first_date)
#df_market_cap.head()<jupyter_output><empty_output><jupyter_text>Market cap weighted<jupyter_code>total_market_cap = df_market_cap.sum(axis=1)
# weights determined in the same month as market cap
df_weights = df_market_cap.div(total_market_cap, axis=0)
# the weights for the index should be determined by past information, 
# i.e.by previous month market_cap
#df_weights_mc = df_weights.shift(1).bfill()
df_weights_mc = df_weights.shift(1)<jupyter_output><empty_output><jupyter_text>Equal weights<jupyter_code>df_temp = (df_weights_mc > 0.0).astype(int)
df_weights_equal = df_temp.div(df_temp.sum(axis=1), axis=0)<jupyter_output><empty_output><jupyter_text>compute index returns<jupyter_code>df_mc_index_returns = pd.DataFrame((df_returns * df_weights_mc).sum(axis=1),columns=['mc_return'])
df_mc_index_returns.dropna(inplace=True)

df_equal_index_returns = pd.DataFrame((df_returns * df_weights_equal).sum(axis=1),columns=['equal_return'])
df_equal_index_returns.dropna(inplace=True)<jupyter_output><empty_output><jupyter_text>Index based on model predictionsBase indices * market cap weighted * equal weighted   Weight generation<jupyter_code>l_base_weights = ['Market Cap', 'Equal']

l_weighting_schemes = ['0']

# Cartesian product tuples
l_weights = list(itertools.product(*[l_base_weights,l_weighting_schemes]))
# WEIGHT ADJUSTMENTS OPTIONS

d_weights = {}

#execution loop

for base_mod, scheme in l_weights:
    print(base_mod, scheme)
    if base_mod == 'Market Cap' :
        df_base_weights =  df_weights_mc.copy()
    if base_mod == 'Equal' :
        df_base_weights =  df_weights_equal.copy()
        
    
    df_mod_weights = df_base_weights.copy()
    
    name_d  = base_mod + ' ' + scheme
    d_weights[name_d] = df_mod_weights<jupyter_output>Market Cap 0
Equal 0<jupyter_text>Build your strategy<jupyter_code># do something better than random

df_rand = pd.DataFrame(np.random.uniform(low=0.0, high=0.01, size=(len(df_weights_mc.index), len(df_weights_mc.columns))), 
                       columns=list(df_weights_mc.columns), index=df_weights_mc.index)
df_w = df_weights_mc + df_rand
df_w = df_w.div(df_w.sum(axis=1), axis=0)
d_weights['Market Cap smart modify'] = df_w

# # SCHEMATIC: CODE DOES NOT EXECUTE LIKE THIS
# # possible loop for training a model
# # and producing df_oos_predictions

# # alternatively use portfolio optimization


# x_names = ['feature1',...]
# y_name = 'returns'

# prediction_dates = df_weights_mc.index[24:]

# for date in prediction_dates:
#     #print(date)
    
#     train_ini_date = date + relativedelta.relativedelta(months=-24)
#     train_final_date = date + relativedelta.relativedelta(months=-1)
    
#     df1 = df_restrict_dates(df_comb_long, train_ini_date, train_final_date)
        
#     df_x_train = df1[x_names].copy()
#     df_y_train = df1[[y_name]].copy()

#     X_train_full = df_x_train.values
#     y_train_full = df_y_train[y_name].values
    
#     model.fit(X_train_full, y_train_full, sample_weight=sample_weights) 
    
            
#     ##### oos results
#     df2 = df_restrict_dates(df_comb_long, date, date)
#     df_x = df2[x_names].copy()    
#     X_oos = df_x.values
    
#     predictions = model.predict(X_oos)
#     df_oos_predictions.loc[date] = predictions
# # SCHEMATIC: CODE DOES NOT EXECUTE LIKE THIS
# # possible loop for weight updates (schematic)
# # based on model predictions df_oos_predictions

# df_base_weights =  df_weights_mc.copy()    
# df_mod_weights = df_base_weights.copy()

    
# for date in prediction_dates:

#     # assume you have made some predictions
#     predictions = df_oos_predictions.loc[date].values 

#     # relate predictions to weight updates
            
#     weights_mod = ....

#     # possibly apply capping rules
    
#     df_mod_weights.loc[date] = weights_mod 

# name_d  = 'xx'
# d_weights[name_d] = df_mod_weights<jupyter_output><empty_output><jupyter_text>Build index  Monthly index levels<jupyter_code># build date frame with indices (no rebalancing)

start_date = datetime.datetime(2009,1,1)
end_date = datetime.datetime(2015,12,31)

starting_level = 100

df_r_in = df_restrict_dates(df_returns, start_date, end_date)
frequency = 'monthly'

for k, name in enumerate(sorted(d_weights.keys())):
    print(name)
    
    df_w_in = df_restrict_dates(d_weights[name], start_date, end_date)
    df_temp  =  index_levels_from_returns(df_w_in, df_r_in, out_field=name, starting_level=starting_level, 
                                          transaction_costs=False, frequency=frequency)
    if k == 0:
        df_i_comb = df_temp
    else:
        df_i_comb = df_i_comb.merge(df_temp, left_index=True, right_index=True)              
# plot without rebalancing costs   
font = {'size'   : 24}

mpl.rc('font', **font)
cm = plt.get_cmap('jet')
#cm = plt.get_cmap('viridis')

sns.set(font_scale=2.5)
sns.set_style("whitegrid")

#fields = 'Equal'
#fields = 'Market Cap'
fields = None

headers = df_i_comb.columns

if fields is not None:
    headers = list(filter(lambda s: fields in s, df_i_comb.columns))
    
df_i_comb[headers].plot(figsize=(20,12), colormap=cm)
plt.title(frequency.title() + ' Index performance (no rebalancing costs)')
print()
extract_performance(df_i_comb[headers])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1034354458638224, 'fraction_numerical': 0.010666005208979288, 'mean_word_length': 5.241486068111455, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '2633', 'license': 'MIT', 'n_tokens_mistral': 2893, 'n_tokens_neox': 2813, 'n_words': 556, 'path': 'notebooks/index_generation_odsc.ipynb', 'repo_name': 'jbauer44/factor_investing', 'size': '8063'}