<jupyter_start><jupyter_text>How to identify dives:- A dive starts and stops approximately a the same hight close to data['Depth'].max()- A dive is not part of a bigger dive- Idea definition: a dive is a max1-min1-max2 triple with abs(max1 - max2) < min(max1-min1, max2-min2), i.e. the depth of a dive is bigger than the difference between the starting and end point<jupyter_code>filter_max = (data['Depth'][:-2].values < data['Depth'][1:-1].values) \
    & (data['Depth'][1:-1].values >= data['Depth'][2:].values)
filter_max = np.insert(filter_max, 0, False)  # padding
filter_max = np.append(filter_max, False)

filter_min = (data['Depth'][:-2].values >= data['Depth'][1:-1].values) \
    & (data['Depth'][1:-1].values < data['Depth'][2:].values)
filter_min = np.insert(filter_min, 0, False)  # padding
filter_min = np.append(filter_min, False)

filter_min[:np.where(filter_max)[0].min()] = False  # make sure we start and end with a max
filter_min[np.where(filter_max)[0].max():] = False
# Nr triples:
sum(filter_min)
# sanity check: are there 2 min without a max in between or vice versa?
# if everything went right this should yield [0, 1] (because we start with a max)
np.unique((1*filter_max - 1*filter_min).cumsum())
plt.hist(data['Depth'][filter_max], 100)
pass
plt.plot(data['DateTime'], data['Depth'])
plt.scatter(data['DateTime'][filter_max], data['Depth'][filter_max], marker='o', c='green')
plt.scatter(data['DateTime'][filter_min], data['Depth'][filter_min], marker='x', c='red')
start = 12000
stop = start + 550
plt.plot(data['DateTime'][start:stop], data['Depth'][start:stop])
plt.scatter(
    data['DateTime'][start:stop][filter_max[start:stop]], 
    data['Depth'][start:stop][filter_max[start:stop]],
    marker='o',
    c='green'
)
plt.scatter(
    data['DateTime'][start:stop][filter_min[start:stop]], 
    data['Depth'][start:stop][filter_min[start:stop]],
    marker='x',
    c='red'
)
plt.ylim(-13, 4)
start = 4200
stop = start + 200
plt.plot(data['DateTime'][start:stop], data['Depth'][start:stop])
plt.scatter(
    data['DateTime'][start:stop][filter_max[start:stop]], 
    data['Depth'][start:stop][filter_max[start:stop]],
    marker='o',
    c='green'
)
plt.scatter(
    data['DateTime'][start:stop][filter_min[start:stop]], 
    data['Depth'][start:stop][filter_min[start:stop]],
    marker='x',
    c='red'
)
plt.ylim(1, 4)
plt.scatter(
    np.minimum(
        data['Depth'][filter_max][:-1].values - data['Depth'][filter_min].values,
        data['Depth'][filter_max][1:].values - data['Depth'][filter_min].values
    ),
    abs(np.diff(data['Depth'][filter_max])), 
    alpha=0.2
)
plt.ylabel('Diff between starting points: abs(max1-max2)')
plt.xlabel('Depth of the dive: min(max1-min1, max2-min2)')
while True:
    triple_depth = np.minimum(
        data['Depth'][filter_max][:-1].values - data['Depth'][filter_min].values,
        data['Depth'][filter_max][1:].values - data['Depth'][filter_min].values
    )
    triple_diff = abs(data['Depth'][filter_max][:-1].values - data['Depth'][filter_max][1:].values)
    smaller_max = np.where(
        data['Depth'][filter_max][:-1].values < data['Depth'][filter_max][1:].values, 
        np.where(filter_max)[0][:-1], np.where(filter_max)[0][1:]
    )

    triple_bad = triple_depth < triple_diff
    print('Nr bad triples: ', sum(triple_bad))
    if not sum(triple_bad):
        break

    filter_max[smaller_max[triple_bad]] = False  # remove smaller max of non-dives

    # fuse mins where there are multiple in a row:
    triple_groups = np.split(np.arange(filter_max.shape[0]), np.where(filter_max)[0])
    smallest_mins = np.array([min(tg, key=lambda t: data['Depth'][t]) for tg in triple_groups])
    filter_min[:] = False
    filter_min[smallest_mins] = True

    filter_min[:np.where(filter_max)[0].min()] = False  # make sure we start and end with a max
    filter_min[np.where(filter_max)[0].max():] = False
# Nr triples:
sum(filter_min)
# sanity check: are there 2 min without a max in between or vice versa?
# if everything went right this should yield [0, 1] (because we start with a max)
np.unique((1*filter_max - 1*filter_min).cumsum())
plt.hist(data['Depth'][filter_max], 100)
pass
plt.plot(data['DateTime'], data['Depth'])
plt.scatter(data['DateTime'][filter_max], data['Depth'][filter_max], marker='o', c='green')
plt.scatter(data['DateTime'][filter_min], data['Depth'][filter_min], marker='x', c='red')
start = 12000
stop = start + 550
plt.plot(data['DateTime'][start:stop], data['Depth'][start:stop])
plt.scatter(
    data['DateTime'][start:stop][filter_max[start:stop]], 
    data['Depth'][start:stop][filter_max[start:stop]],
    marker='o',
    c='green'
)
plt.scatter(
    data['DateTime'][start:stop][filter_min[start:stop]], 
    data['Depth'][start:stop][filter_min[start:stop]],
    marker='x',
    c='red'
)
plt.ylim(-13, 4)
start = 4200
stop = start + 200
plt.plot(data['DateTime'][start:stop], data['Depth'][start:stop])
plt.scatter(
    data['DateTime'][start:stop][filter_max[start:stop]], 
    data['Depth'][start:stop][filter_max[start:stop]],
    marker='o',
    c='green'
)
plt.scatter(
    data['DateTime'][start:stop][filter_min[start:stop]], 
    data['Depth'][start:stop][filter_min[start:stop]],
    marker='x',
    c='red'
)
plt.ylim(1, 4)
plt.scatter(
    np.minimum(
        data['Depth'][filter_max][:-1].values - data['Depth'][filter_min].values,
        data['Depth'][filter_max][1:].values - data['Depth'][filter_min].values
    ),
    abs(np.diff(data['Depth'][filter_max])), 
    alpha=0.2
)
plt.ylabel('Diff between starting points: abs(max1-max2)')
plt.xlabel('Depth of the dive: min(max1-min1, max2-min2)')
# distribution of dive length in seconds
plt.hist(np.diff(np.where(filter_max)[0]), 200)
pass
# 90% of the curves are shorter than 16 seconds
sum(np.diff(np.where(filter_max)[0]) <= 16) / sum(filter_min)
# 0%, 5%, 10%, ... , 95%, 100% quantiles
np.quantile(np.diff(np.where(filter_max)[0]), np.arange(21)/20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19581939799331102, 'fraction_numerical': 0.023411371237458192, 'mean_word_length': 5.24321503131524, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '333175', 'license': 'MIT', 'n_tokens_mistral': 2203, 'n_tokens_neox': 2146, 'n_words': 404, 'path': 'diving.ipynb', 'repo_name': 'ArneKramerSunderbrink/functional_data_hmm', 'size': '5980'}