<jupyter_start><jupyter_text>Correlation Matrix<jupyter_code>import pandas as pd
import glob<jupyter_output><empty_output><jupyter_text>Get proper data files to pandas dataframe<jupyter_code>wig20 = ['MBANK', 'SANPL.', 'CCC.', 'KGHM', 'LOTOS', 'LPP', 'CDPROJEKT', 'PEKAO',
         'PGNIG', 'PKNORLEN', 'PKOBP', 'ORANGEPL', 'CYFRPLSAT', 'PGE.', 'PZU.',
         'TAURONPE', 'JSW.', 'ALIOR', 'DINOPL', 'PLAY.']

mwig40 = ['AMICA', 'AMREST', 'ASSECOPOL', 'BUDIMEX', 'CIECH', 'COMARCH', 'DOMDEV', 'ECHO',
          'EUROCASH', 'FAMUR', 'FORTE', 'GETIN.', 'GTC', 'HANDLOWY', 'INGBSK', 'INTERCARS',
          'KETY', 'DEVELIA', 'MILLENNIUM', 'STALPROD', 'NEUCA', 'VRG', 'KERNEL', 'GRUPAAZOTY',
          'ENEA', 'BOGDANKA', 'ASSECOSEE', 'MABION', '11BIT', 'GPW', 'BENEFIT', 'KRUK',
          'PKPCARGO', 'ENERGA', 'LIVECHAT', 'WIRTUALNA', 'BNPPPL', 'CLNPHARMA', 'PLAYWAY',
          'TSGAMES']

swig80 = ['ABPL', 'AGORA', 'AMBRA', 'APATOR', 'ASTARTA', 'ATMGRUPA', 'BIOTON', 'BORYSZEW', 'BOS.',
          'COMP.', 'DEBICA', 'INSTALKRK', 'KOGENERA', 'KRUSZAWICA', 'LENTEX', 'MCI', 'MENNICA',
          'NETIA', 'PEP.', 'POLICE.', 'POLIMEXMS', 'RAFAKO', 'SANOK', 'SNIEZKA', 'STALEXP', 'TIM',
          'WAWEL', 'MANGATA', 'OPONEO.PL', 'RAINBOW', 'ASBIS', 'ASSECOBS', 'CIGAMES', 'WIELTON',
          'TRAKCJA', 'UNIBEP', 'BAHOLDING', 'QUERCUS', 'PGSSOFT', 'MIRBUD', 'EKOEXPORT', 'ARCTIC',
          'FERRO', 'SERINUS', 'VOTUM', 'IMCOMPANY', 'OVOSTAR', 'RYVU', 'ACAUTOGAZ', 'TOYA', 
          'VOXEL', 'MEDICALG', 'ELEMENTAL', 'DATAWALK', 'UNIMOT', 'ZEPAK', 'INTERAOLT', 'PHN',
          'MLPGROUP', 'MERCATOR', 'NEWAG', 'CPGROUP', 'PCCROKITA', 'TORPOL', 'ALUMETAL', 'VIGOSYS',
          'IDEABANK', 'ATAL', 'PEKABEX', 'PRAIRIE', 'ENTER', 'ARCHICOM', 'PBKM', 'AUTOPARTN', 'R22.',
          'OAT', 'ULTGAMES', 'MLSYSTEM', 'BOOMBIT', 'SELVITA']

combined = wig20 + mwig40 + swig80

all_files = glob.glob(r"..\data\*.mst")
found_files = []
for path in all_files:
    for symbol in combined:
        if symbol in path and "RCFL" not in path and "RCFS" not in path and "RCRKE" not in path:
            found_files.append(path)
found_files<jupyter_output><empty_output><jupyter_text>Correlation matrix based on close prices<jupyter_code>close_pp_df = pd.DataFrame([])
close_pp_df.index = pd.to_datetime(close_pp_df.index, format="%Y%m%d")
start_date = "20180101"
output_file = "corr_matrix_close.csv"

for path in found_files:
    temp_df = pd.read_csv(path, index_col=1)
    temp_df.index = pd.to_datetime(temp_df.index, format="%Y%m%d")
    temp_df = temp_df.loc[temp_df.index >= start_date]
    temp_df["CLOSE_%"] = temp_df["<CLOSE>"] / temp_df["<CLOSE>"].values[0]
    try:
        close_pp_df[temp_df["<TICKER>"].values[0]] = temp_df["CLOSE_%"]
    except ValueError:
        print("Failed for: " + temp_df["<TICKER>"].values[0])
        continue
correlation_matrix = close_pp_df.corr()
correlation_matrix.to_csv("..\\output_data\\"+output_file)
correlation_matrix["CDPROJEKT"]<jupyter_output>Failed for: BAHOLDING<jupyter_text>Correlation matrix based on MA<jupyter_code>close_pp_df = pd.DataFrame([])
close_pp_df.index = pd.to_datetime(close_pp_df.index, format="%Y%m%d")
start_date = "20180101"
ma_period = 11
output_file = "corr_matrix_ma.csv"

for path in found_files:
    temp_df = pd.read_csv(path, index_col=1)
    temp_df.index = pd.to_datetime(temp_df.index, format="%Y%m%d")
    temp_df['MA'] = temp_df["<CLOSE>"].rolling(11).mean();
    temp_df = temp_df.loc[temp_df.index >= start_date]
    temp_df["MA_%"] = temp_df["MA"] / temp_df["MA"].values[0]
    try:
        close_pp_df[temp_df["<TICKER>"].values[0]] = temp_df["MA_%"]
    except ValueError:
        print("Failed for: " + temp_df["<TICKER>"].values[0])
        continue
correlation_matrix = close_pp_df.corr()
correlation_matrix.to_csv("..\\output_data\\"+output_file)
correlation_matrix["CDPROJEKT"]<jupyter_output>Failed for: BAHOLDING
{'fraction_non_alphanumeric': 0.19856336582863007, 'fraction_numerical': 0.011287839917906618, 'mean_word_length': 4.7507374631268435, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '340452', 'license': 'MIT', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1614, 'n_words': 282, 'path': 'examples/notebooks/20_AnalysisCorelationMatrix.ipynb', 'repo_name': 'WojciechLepiarz/StockBee', 'size': '3898'}