<jupyter_start><jupyter_text>This file contains the code which was used to select and time-shift data from Artemis for the two larger Artemis csv files.<jupyter_code>import numpy as np
import pandas as pd
import datetime
import matplotlib.pyplot as plt

posFrame = pd.read_csv('csv files/POS4Artemis1Year.csv')
windFrame = pd.read_csv('csv files/SW4Artemis1Year.csv')
windFrame = windFrame.loc[:, ~windFrame.columns.str.contains('^Unnamed')]
posFrame = posFrame.loc[:, ~posFrame.columns.str.contains('^Unnamed')]
windFrame['EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ'] = pd.to_datetime(windFrame['EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ'])
windFrame['date'] = pd.to_datetime(windFrame['EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ']).dt.date
posFrame['EPOCH_yyyy-mm-ddThh:mm:ss.sssZ'] = pd.to_datetime(posFrame['EPOCH_yyyy-mm-ddThh:mm:ss.sssZ'])
posFrame['date'] = pd.to_datetime(posFrame['EPOCH_yyyy-mm-ddThh:mm:ss.sssZ']).dt.date
mergedFrame = pd.merge(windFrame, posFrame, on='date')
mergedFrame = mergedFrame.drop(['date', 'EPOCH_yyyy-mm-ddThh:mm:ss.sssZ'], axis=1)
mergedFrame = mergedFrame[mergedFrame.ARTEMIS_MAG_AVG_nt != -1.0E+31]
mergedFrame = mergedFrame[mergedFrame.ARTEMIS_DENSITY_Ncc != -1.0E+31]
mergedFrame = mergedFrame[mergedFrame.ARTEMIS_SPEED_kms != -1.0E+31]

mergedFrame = mergedFrame[mergedFrame['GSE_X_RE'] >= -35]
mergedFrame = mergedFrame[mergedFrame['SSE_X_RM'] >= 0]
mergedFrame = mergedFrame[mergedFrame['ARTEMIS_DIST_AU'] >= 1]
mergedFrame = mergedFrame.resample('H', on = 'EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ').mean().reset_index()

df2 = pd.read_csv('csv files/Omni4Artemis1Year.csv')
df2['EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ'] = pd.to_datetime(df2['EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ'])
df2 = df2[df2['OMNI_MAG_AVG_nT'] != -1.0E+31]
df2 = df2[df2['OMNI_SPEED_kms'] != -1.0E+31]
df2 = df2[df2['OMNI_DENSITY_Ncc'] != -1.0E+31]
df2 = df2.resample('H', on = 'EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ').mean().reset_index()

mergedFrame = mergedFrame.drop(['GSE_X_RE', 'GSE_Y_RE', 'GSE_Z_RE', 'SSE_X_RM', 'SSE_Y_RM', 'SSE_Z_RM'], axis=1)
mergedFrame['new_time'] = mergedFrame['EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ'] - pd.Timedelta(hours = 1)
mergedFrame['Time_offset_hours'] = 1

mF = pd.merge(df2, mergedFrame, how='right', left_on='EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ', right_on='new_time')
mF = mF.dropna(axis=0, how='any', thresh=None, subset=None, inplace=False)


fig1 = plt.figure(1)
fig1.suptitle("Artemis speed compared with OMNI speed")
ax = plt.gca()
mF.plot(kind='line', x='EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ', y='OMNI_SPEED_kms', color='red', ax=ax)
mF.plot(kind='line', x='EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ', y='ARTEMIS_SPEED_kms', ax=ax)
ax.set_xlim([datetime.date(2018, 3, 15), datetime.date(2018, 11, 1)])

fig2 = plt.figure(2)
fig2.suptitle("Artemis density compared with OMNI density")
ay = plt.gca()
mF.plot(kind='line', x='EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ', y='OMNI_DENSITY_Ncc', color='red', ax=ay)
mF.plot(kind='line', x='EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ', y='ARTEMIS_DENSITY_Ncc', ax=ay)
ay.set_xlim([datetime.date(2018, 3, 15), datetime.date(2018, 11, 1)])

fig3 = plt.figure(3)
fig3.suptitle("Artemis magnetic field average compared with OMNI magnetic field average")
az = plt.gca()
mF.plot(kind='line', x='EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ', y='OMNI_MAG_AVG_nT', color='red', ax=az)
mF.plot(kind='line', x='EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ', y='ARTEMIS_MAG_AVG_nt', ax=az)
az.set_xlim([datetime.date(2018, 3, 15), datetime.date(2018, 11, 1)])

fig4 = plt.figure(4)
fig4.suptitle("Artemis distance from the Sun in AU")
az = plt.gca()
mF.plot(kind='line', x='EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ', y='ARTEMIS_DIST_AU', color='red', ax=az)
az.set_xlim([datetime.date(2018, 3, 15), datetime.date(2018, 11, 1)])

plt.show()

mF.describe()

#mF.to_csv('Artemis1Year.csv')
import numpy as np
import pandas as pd
import datetime
import matplotlib.pyplot as plt

posFrame = pd.read_csv('csv files/POS4Artemis1Year2.csv')
windFrame = pd.read_csv('csv files/SW4Artemis1Year2.csv')
windFrame = windFrame.loc[:, ~windFrame.columns.str.contains('^Unnamed')]
posFrame = posFrame.loc[:, ~posFrame.columns.str.contains('^Unnamed')]
windFrame['EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ'] = pd.to_datetime(windFrame['EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ'])
windFrame['date'] = pd.to_datetime(windFrame['EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ']).dt.date
posFrame['EPOCH_yyyy-mm-ddThh:mm:ss.sssZ'] = pd.to_datetime(posFrame['EPOCH_yyyy-mm-ddThh:mm:ss.sssZ'])
posFrame['date'] = pd.to_datetime(posFrame['EPOCH_yyyy-mm-ddThh:mm:ss.sssZ']).dt.date
mergedFrame = pd.merge(windFrame, posFrame, on='date')
mergedFrame = mergedFrame.drop(['date', 'EPOCH_yyyy-mm-ddThh:mm:ss.sssZ'], axis=1)
mergedFrame = mergedFrame[mergedFrame.ARTEMIS_MAG_AVG_nt != -1.0E+31]
mergedFrame = mergedFrame[mergedFrame.ARTEMIS_DENSITY_Ncc != -1.0E+31]
mergedFrame = mergedFrame[mergedFrame.ARTEMIS_SPEED_kms != -1.0E+31]

mergedFrame = mergedFrame[mergedFrame['GSE_X_RE'] >= -35]
mergedFrame = mergedFrame[mergedFrame['SSE_X_RM'] >= 0]
mergedFrame = mergedFrame[mergedFrame['ARTEMIS_DIST_AU'] >= 1]
mergedFrame = mergedFrame.resample('H', on = 'EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ').mean().reset_index()

df2 = pd.read_csv('csv files/Omni4Artemis1Year2.csv')
df2['EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ'] = pd.to_datetime(df2['EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ'])
df2 = df2[df2['OMNI_MAG_AVG_nT'] != -1.0E+31]
df2 = df2[df2['OMNI_SPEED_kms'] != -1.0E+31]
df2 = df2[df2['OMNI_DENSITY_Ncc'] != -1.0E+31]
df2 = df2.resample('H', on = 'EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ').mean().reset_index()

mergedFrame = mergedFrame.drop(['GSE_X_RE', 'GSE_Y_RE', 'GSE_Z_RE', 'SSE_X_RM', 'SSE_Y_RM', 'SSE_Z_RM'], axis=1)
mergedFrame['new_time'] = mergedFrame['EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ'] - pd.Timedelta(hours = 1)
mergedFrame['Time_offset_hours'] = 1

mF = pd.merge(df2, mergedFrame, how='right', left_on='EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ', right_on='new_time')
mF = mF.dropna(axis=0, how='any', thresh=None, subset=None, inplace=False)


fig1 = plt.figure(1)
fig1.suptitle("Artemis speed compared with OMNI speed")
ax = plt.gca()
mF.plot(kind='line', x='EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ', y='OMNI_SPEED_kms', color='red', ax=ax)
mF.plot(kind='line', x='EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ', y='ARTEMIS_SPEED_kms', ax=ax)
#ax.set_xlim([datetime.date(2018, 3, 15), datetime.date(2018, 11, 1)])

fig2 = plt.figure(2)
fig2.suptitle("Artemis density compared with OMNI density")
ay = plt.gca()
mF.plot(kind='line', x='EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ', y='OMNI_DENSITY_Ncc', color='red', ax=ay)
mF.plot(kind='line', x='EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ', y='ARTEMIS_DENSITY_Ncc', ax=ay)
#ay.set_xlim([datetime.date(2018, 3, 15), datetime.date(2018, 11, 1)])

fig3 = plt.figure(3)
fig3.suptitle("Artemis magnetic field average compared with OMNI magnetic field average")
az = plt.gca()
mF.plot(kind='line', x='EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ', y='OMNI_MAG_AVG_nT', color='red', ax=az)
mF.plot(kind='line', x='EPOCH_TIME_yyyy-mm-ddThh:mm:ss.sssZ', y='ARTEMIS_MAG_AVG_nt', ax=az)
#az.set_xlim([datetime.date(2018, 3, 15), datetime.date(2018, 11, 1)])

fig4 = plt.figure(4)
fig4.suptitle("Artemis distance from the Sun in AU")
az = plt.gca()
mF.plot(kind='line', x='EPOCH_TIME__yyyy-mm-ddThh:mm:ss.sssZ', y='ARTEMIS_DIST_AU', color='red', ax=az)
#az.set_xlim([datetime.date(2018, 3, 15), datetime.date(2018, 11, 1)])

plt.show()

mF.describe()

#mF.to_csv('Artemis1Year2.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1878246969495138, 'fraction_numerical': 0.0333022512321833, 'mean_word_length': 7.6002290950744555, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '634777', 'license': 'MIT', 'n_tokens_mistral': 3582, 'n_tokens_neox': 3340, 'n_words': 445, 'path': 'Data Processing/Artemis1Year.ipynb', 'repo_name': 'jamesniro/Prediction-Solar-Wind-with-Machine-Learning', 'size': '7507'}