<jupyter_start><jupyter_text>Script to check stylised facts of the parallel simulation  0. Initialise packages and pull data<jupyter_code># change root directory
import os
os.chdir("C:\\Users\\chris\\Documents\\git\\abides\\")
current_dir = "C:\\Users\\chris\\Documents\\git\\abides\\util\\plotting\\phd\\grid_simulation_002\\"
# Call packages
import os
import numpy as np 
import pandas as pd 
import scipy.stats as stats
import matplotlib.pyplot as plt
from util.formatting.mid_price_from_orderbook import save_mid_price
from sklearn import preprocessing
from glob import glob
"""
Set folder location and initialise time parameters
"""

# Actual data location
actual_data_folder = "data/20210312btcusd/"

# Simulation folder location
simulation_log_folder = "C:/Users/chris/Documents/git/abides/log/20210312PARALLEL/*/"

# Timespan of the analysis
starttime = pd.to_datetime('2021-03-12 09:00:00')
endtime = pd.to_datetime('2021-03-12 16:00:00')

# Resample frequency
resample_freq = "1T"

"""
Actual data
"""

# Read data
mid_price = pd.read_pickle(actual_data_folder + "ts.bz2")
order_stream = pd.read_csv( actual_data_folder + "order.csv")

# Convert to unix time
order_stream.index = pd.to_datetime(order_stream.time ,unit='ms')
order_stream.time = pd.to_datetime(order_stream.time ,unit='ms')

# Identify data that falls within predetermined start/end time
mid_price = mid_price[(mid_price.index >= starttime) & (mid_price.index <= endtime)]
order_stream = order_stream[(order_stream.index >= starttime) & (order_stream.index <= endtime)]

# Resample price and quantity to obtain snapshots information
order_resample = order_stream[["price", "qty"]].resample(resample_freq)

open_price = order_resample["price"].first().ffill()
total_volume = order_resample["qty"].sum()

order_snapshot = pd.DataFrame({"price": open_price,
                               "qty": total_volume
                               })

# check for incomplete data

for current_sim_folder in glob(simulation_log_folder):

    if len(glob(current_sim_folder + "*")) < 1000:

        print("\nCurrent location: ", current_sim_folder)
        print("Number of files: ", len(glob(current_sim_folder + "*")))
"""
Simulated data - N.B. This takes ages and does not need to be run again. Last ran on 2021/05/04
"""

# For all simulations, retrieve and save simulated midprice data using prebuilt utility in ABIDES
# N.B. Inputs here are (file_loc, output_loc)
for current_sim_folder in glob(simulation_log_folder):

    save_mid_price(current_sim_folder + "ORDERBOOK_BTC_FREQ_T.bz2", current_sim_folder)
# loop for all simulations
for current_sim_folder in glob(simulation_log_folder):

    # identify agent combinations
    _, _, ZI, ZIP, MR, MMT, _ = current_sim_folder.split("_")

    # pull data from each simulation
    sim_mid_price = pd.read_pickle(current_sim_folder + "ORDERBOOK_BTC_FREQ_T_mid_price.bz2")
    sim_exchange = pd.read_pickle(current_sim_folder + "ExchangeAgent0.bz2")
    
    # filter for trades
    sim_exchange = sim_exchange[sim_exchange['EventType'] == 'LAST_TRADE']

    # Create simulated order stream using the data from the exchange agent
    if len(sim_exchange) <= 0:
        print("There appear to be no simulated trades.")
        sys.exit()

    sim_exchange['PRICE'] = [y for x,y in sim_exchange['Event'].str.split(',')]
    sim_exchange['SIZE'] = [x for x,y in sim_exchange['Event'].str.split(',')]
    sim_exchange['PRICE'] = sim_exchange['PRICE'].str.replace('$','').astype('float64')
    sim_exchange['SIZE'] = sim_exchange['SIZE'].astype('float64')

    # Populate simulated order stream
    sim_order_stream = pd.DataFrame({"price": sim_exchange.PRICE,
                                    "qty": sim_exchange.SIZE,
                                    "Time": sim_exchange.index
                                    })

    sim_order_stream.index = sim_exchange.index

    # Identify simulated data that falls within predetermined start/end time
    sim_mid_price = sim_mid_price[(sim_mid_price.index >= starttime) & (sim_mid_price.index <= endtime)]
    sim_order_stream = sim_order_stream[(sim_order_stream.index >= starttime) & (sim_order_stream.index <= endtime)]

    # Resample simulated price and quantity to obtain snapshots information
    sim_order_resample = sim_order_stream[["price", "qty"]].resample(resample_freq)

    sim_open_price = sim_order_resample["price"].first().ffill()
    sim_total_volume = sim_order_resample["qty"].sum()

    sim_order_snapshot = pd.DataFrame({"price": sim_open_price,
                                "qty": sim_total_volume
                                })

    # create a dataframe of current simulation
    current_df = pd.DataFrame([[ZI, ZIP, MR, MMT, sim_mid_price, sim_order_snapshot]])

    # loop to append dataframe from each simulation into a single dataframe
    if current_sim_folder == glob(simulation_log_folder)[0]:

        sim_df = current_df

    else:

        sim_df = sim_df.append(current_df)


# once the single dataframe is created, label columns    
sim_df.columns = ["ZI", "ZIP", "MR", "MMT", "midprice", "orderbook"]

sim_df<jupyter_output><empty_output><jupyter_text>1. Asset returns exhibit heavy tails and aggregation normality (high kurtosis)I believe that the studies suggest a normal distribution with high kurtosis---**Our findings:**- Initial findings were fine, albeit with too low variance    - Maybe due to the market maker    - Maybe due to the high precision when querying historic price    - Maybe due to insufficient count of technical agents- Tuning the agents to receive slightly noisier observations made simulation look much closer to reality- Maybe we should formally test the kurtosis of these two distributions- Simulation replicates the data well now<jupyter_code>"""
Initialise
"""

asset_return_interval = 1 # 1 minute asset returnsreturns
"""
Simulated data
"""

log_returns = []

for sim_mid_price in sim_df.midprice:

    # calculate returns
    sim_log_returns = np.log(np.array((sim_mid_price[asset_return_interval:]))/np.array((sim_mid_price[:-asset_return_interval])))

    # append result to log_returns array
    log_returns.append(sim_log_returns)


sim_df.insert(loc=len(sim_df.columns), column="log_returns", value=log_returns)
sim_df.midprice.iloc[20]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10665610142630745, 'fraction_numerical': 0.01220285261489699, 'mean_word_length': 4.118410381184104, 'pattern_counts': {'":': 7, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '344600', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2042, 'n_tokens_neox': 1923, 'n_words': 495, 'path': 'util/plotting/phd/grid_simulation_001/grid_simulation_001.ipynb', 'repo_name': 'chris-jh-cho/abides', 'size': '6310'}