<jupyter_start><jupyter_text>Massage YNDX Data Short notebook used to do some massaging of the YNDX-dataNot necessary to use to get this repo to work<jupyter_code>import os
import pandas as pd

import numpy as np
import matplotlib.pyplot as plt
BASE_URL = '/content/drive/My Drive/NTNU/6. Semester/NEURO140/Data/1M/YNDX'
f15 = 'YNDX_150101_151231.csv'
f16 = 'YNDX_160101_161231.csv'
filepath15 = os.path.join(BASE_URL, f15)
df15 = pd.read_csv(filepath15)
filepath16 = os.path.join(BASE_URL, f16)
df16 = pd.read_csv(filepath16)
df16.head()
# (df15['close'].iloc[-1] - df15['open'].iloc[0]) / df15['open'].iloc[0]
import datetime
print(datetime.datetime.fromtimestamp(1585526400))
print(datetime.datetime.fromtimestamp(1585612800))
cum_vol = 0
cum_pv = 0
last_day = None

for index, row in df16.iterrows():
    curr_day = str(int(row['<DATE>']))[-1]
    if curr_day != last_day:
        last_day = curr_day
        cum_vol = 0
        cum_pv = 0

    pv = sum(row[['high', 'low', 'close']]) / 3

    cum_vol += row['Volume']
    cum_pv += pv * row['Volume']

    if cum_vol == 0:
        vwap = 0
    else:
        vwap = cum_pv / cum_vol

    if index % 1000 == 0:
        print(f"Doing index {index}, vwap {vwap} cum_vol {cum_vol}")
    if (index + 1) % 2000 == 0:
        pass

    df16.at[index, 'vwap'] = vwap


df16.head(n=100)
f15 = 'YNDX_15_alt.csv'
f16 = 'YNDX_16_alt.csv'

filepath15 = os.path.join(BASE_URL, f15)
df15.to_csv(filepath15)

filepath16 = os.path.join(BASE_URL, f16)
df16.to_csv(filepath16)
%matplotlib inline
import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (25,10)
import random
small_df1 = df16.iloc[random.sample(df16.index.values.tolist(), 60)]
small_df2 = df16.iloc[random.sample(df16.index.values.tolist(), 60)]

plt.plot(df16['close'], zorder=-100000)
plt.scatter(small_df1.index.values.tolist(), small_df1['close'], marker='^', color='xkcd:lime green', s=80)
plt.scatter(small_df2.index.values.tolist(), small_df2['close'], marker='v', color='xkcd:pink red', s=80)
plt.show();
df15.shape[0] + df16.shape[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15658021133525457, 'fraction_numerical': 0.08261287223823247, 'mean_word_length': 4.368556701030927, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '431988', 'license': 'MIT', 'n_tokens_mistral': 960, 'n_tokens_neox': 851, 'n_words': 168, 'path': 'utility-notebooks/massage_yandex.ipynb', 'repo_name': 'ankile/reinforcement-trading', 'size': '2082'}