<jupyter_start><jupyter_text>BPR Data - Difference plots   API Information Setup<jupyter_code>import pandas as pd
import hvplot.pandas
import numpy as np
import matplotlib.dates as dates
import warnings
warnings.filterwarnings('ignore')
import datetime
def dateparse (date_string):
    return datetime.datetime.strptime(date_string, '%m/%d/%Y %H:%M:%S')<jupyter_output><empty_output><jupyter_text>Nemo Center 2000-2002<jupyter_code>!head /home/jovyan/data/bpr/Axial_Deformation/nemo2000-2002-BPR-center-15sec-driftcorr-detided-lpf.txt
# bpr_file = '/home/jovyan/data/bpr/Axial_Deformation/nemo2000-2002-BPR-center-15sec-driftcorr-detided-lpf.txt
# df_nemoC = pd.read_csv(bpr_file, parse_dates=True, date_parser=dateparse, index_col='Date',
#                        dtype = {'Date': object,'RawDep': np.float64,
#                                 'Temp': np.float64, 'DriftCorrRawDep': np.float64,
#                                 'SpotlDep': np.float64, 'DriftCorrSpotlDep': np.float64,
#                                 'LPFDep,DriftCorrLPFDep': np.float64})
# del df_nemoC['Temp']
# df_nemoC.columns
# df_nemoC.head()
# df_nemoC_sub=df_nemoC[df_nemoC.RawDep > 1500]
# df_nemoC_sub.hvplot.scatter(x='Date', y= ['DriftCorrRawDep', 'DriftCorrSpotlDep', 'DriftCorrLPFDep'],
#                         subplots= True,
#                         datashade=True, 
#                         flip_yaxis=True).cols(1)<jupyter_output><empty_output><jupyter_text>BPR Data - Difference plots NEMO 2007-2010  Nemo Center 2007-2010<jupyter_code>bpr_file = '/home/jovyan/data/bpr/Axial_Deformation/nemo2007-2010-BPR-center-15sec-driftcorr-detided-lpf.txt'
df_nemoC = pd.read_csv(bpr_file, parse_dates=True, date_parser=dateparse, index_col='Date',
                       dtype = {'Date': object,'RawDep': np.float64,
                                'Temp': np.float64, 'DriftCorrRawDep': np.float64,
                                'SpotlDep': np.float64, 'DriftCorrSpotlDep': np.float64,
                                'LPFDep,DriftCorrLPFDep': np.float64})
!head /home/jovyan/data/botpt/bpr_files/Axial_Deformation/nemo2007-2010-BPR-center-15sec-driftcorr-detided-lpf.txt
df_nemoC.head()
df_nemoC.hvplot.scatter(x='Date', y= 'RawDep', datashade=True, flip_yaxis=True)<jupyter_output><empty_output><jupyter_text>Nemo South 2009-2011<jupyter_code>bpr_file = '/home/jovyan/data/bpr/Axial_Deformation/nemo2009-2011-BPR-south-15sec-detided-lpf.txt'
!head /home/jovyan/data/bpr/Axial_Deformation/nemo2009-2011-BPR-south-15sec-detided-lpf.txt
df_nemoS = pd.read_csv(bpr_file, parse_dates=True, date_parser=dateparse, index_col='Date',
                       dtype = {'Date': object,'Depth': np.float64,
                                'Temp': np.float64, 'SpotlDetidedDepth': np.float64,
                                'LPFDetidedDepth': np.float64})
df_nemoS.head()<jupyter_output><empty_output><jupyter_text>Notes;<jupyter_code>df_nemoS.hvplot.scatter(x='Date', y= ['Depth', 'SpotlDetidedDepth'], subplots= True, datashade=True, flip_yaxis=True).cols(1)<jupyter_output><empty_output><jupyter_text>Difference plot: Nemo Centeral - Nemo South  Nemo South does not have drift corrected values; so I am not sure if this is as valuable... but I do not see these "Pauses" in the older data.<jupyter_code>test = pd.merge(df_nemoC, df_nemoS, how ='outer', indicator=True, left_index=True, right_index=True, suffixes=('_C', '_S'))
df_nemoMerge = test[test['_merge'] == 'both']
del df_nemoMerge['_merge']
df_nemoMerge.head()<jupyter_output><empty_output><jupyter_text>Calculate Depth difference<jupyter_code>depthDiff = df_nemoMerge['RawDep'].values - df_nemoMerge['Depth'].values
df_nemoMerge['diff'] = depthDiff
df_nemoMerge.hvplot.scatter(x='Date', y='diff', datashade=True, flip_yaxis=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13405319989465367, 'fraction_numerical': 0.034237555965235714, 'mean_word_length': 3.9133247089262615, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '382168', 'license': 'MIT', 'n_tokens_mistral': 1534, 'n_tokens_neox': 1398, 'n_words': 194, 'path': 'notebooks/bpr_difference.ipynb', 'repo_name': 'daxsoule/BOTPT', 'size': '3797'}