<jupyter_start><jupyter_text>Time Series DatabaseThis notebook demonstrates the persistent behavior of the database.  Initialization * Clear the file system for demonstration purposes.<jupyter_code># database parameters
ts_length = 100
data_dir = '../db_files'
db_name = 'default'
dir_path = data_dir + '/' + db_name + '/'
# clear file system for testing
if not os.path.exists(dir_path):
    os.makedirs(dir_path)
filelist = [dir_path + f for f in os.listdir(dir_path)]
for f in filelist:
    os.remove(f)<jupyter_output><empty_output><jupyter_text>* Load the database server.<jupyter_code># when running from the terminal
# python go_server_persistent.py --ts_length 100 --db_name 'demo'

# here we load the server as a subprocess for demonstration purposes
server = subprocess.Popen(['python', '../go_server_persistent.py',
                           '--ts_length', str(ts_length), '--data_dir', data_dir, '--db_name', db_name])
time.sleep(5)  # make sure it loads completely<jupyter_output><empty_output><jupyter_text>* Load the database webserver.<jupyter_code># when running from the terminal
# python go_webserver.py

# here we load the server as a subprocess for demonstration purposes
webserver = subprocess.Popen(['python', '../go_webserver.py'])
time.sleep(5)  # make sure it loads completely<jupyter_output><empty_output><jupyter_text>* Import the web interface and initialize it.<jupyter_code>from webserver import *
web_interface = WebInterface()<jupyter_output><empty_output><jupyter_text>Generate Data Let's create some dummy data to aid in our demonstration. You will need to import the `timeseries` package to work with the TimeSeries format.**Note:** the database is persistent, so can store data between sessions, but we will start with an empty database here for demonstration purposes.<jupyter_code>from timeseries import *
def tsmaker(m, s, j):
    '''
    Helper function: randomly generates a time series for testing.

    Parameters
    ----------
    m : float
        Mean value for generating time series data
    s : float
        Standard deviation value for generating time series data
    j : float
        Quantifies the "jitter" to add to the time series data

    Returns
    -------
    A time series and associated meta data.
    '''

    # generate metadata
    meta = {}
    meta['order'] = int(np.random.choice(
        [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]))
    meta['blarg'] = int(np.random.choice([1, 2]))

    # generate time series data
    t = np.arange(0.0, 1.0, 0.01)
    v = norm.pdf(t, m, s) + j * np.random.randn(ts_length)

    # return time series and metadata
    return meta, TimeSeries(t, v)
# generate sample time series
num_ts = 50
mus = np.random.uniform(low=0.0, high=1.0, size=num_ts)
sigs = np.random.uniform(low=0.05, high=0.4, size=num_ts)
jits = np.random.uniform(low=0.05, high=0.2, size=num_ts)

# initialize dictionaries for time series and their metadata
primary_keys = []
tsdict = {}
metadict = {}

# fill dictionaries with randomly generated entries for database
for i, m, s, j in zip(range(num_ts), mus, sigs, jits):
    meta, tsrs = tsmaker(m, s, j)  # generate data
    pk = "ts-{}".format(i)  # generate primary key
    primary_keys.append(pk) # keep track of all primary keys
    tsdict[pk] = tsrs  # store time series data
    metadict[pk] = meta  # store metadata
    
# to assist with later testing
ts_keys = sorted(tsdict.keys())
    
# randomly choose time series as vantage points
num_vps = 5
vpkeys = list(np.random.choice(ts_keys, size=num_vps, replace=False))
vpdist = ['d_vp_{}'.format(i) for i in vpkeys]<jupyter_output><empty_output><jupyter_text>Insert DataLet's start by loading the data into the database, using the REST API web interface.<jupyter_code># check that the database is empty
web_interface.select()
# add stats trigger
web_interface.add_trigger('stats', 'insert_ts', ['mean', 'std'], None)
# insert the time series
for k in tsdict:
    web_interface.insert_ts(k, tsdict[k])
# upsert the metadata
for k in tsdict:
    web_interface.upsert_meta(k, metadict[k])
# add the vantage points
for i in range(num_vps):
    web_interface.insert_vp(vpkeys[i])<jupyter_output><empty_output><jupyter_text>Inspect DataLet's inspect the data, to make sure that all the previous operations were successful.<jupyter_code># select all database entries; all metadata fields
results = web_interface.select(fields=[])

# we have the right number of database entries
assert len(results) == num_ts

# we have all the right primary keys
assert sorted(results.keys()) == ts_keys
# check that all the time series and metadata matches
for k in tsdict:
    results = web_interface.select(fields=['ts'], md={'pk': k})
    assert results[k]['ts'] == tsdict[k]
    results = web_interface.select(fields=[], md={'pk': k})
    for field in metadict[k]:
        assert metadict[k][field] == results[k][field]
# check that the vantage points match
print('Vantage points selected:', vpkeys)
print('Vantage points in database:',
      web_interface.select(fields=None, md={'vp': True}, additional={'sort_by': '+pk'}).keys())
# check that the vantage point distance fields have been created
print('Vantage point distance fields:', vpdist)
web_interface.select(fields=vpdist, additional={'sort_by': '+pk', 'limit': 1})
# check that the trigger has executed as expected (allowing for rounding errors)
for k in tsdict:
    results = web_interface.select(fields=['mean', 'std'], md={'pk': k})
    assert np.round(results[k]['mean'], 4) == np.round(tsdict[k].mean(), 4)
    assert np.round(results[k]['std'], 4) == np.round(tsdict[k].std(), 4)<jupyter_output><empty_output><jupyter_text>Let's generate an additional time series for similarity searches. We'll store the time series and the results of the similarity searches, so that we can compare against them after reloading the database.<jupyter_code>_, query = tsmaker(np.random.uniform(low=0.0, high=1.0),
                   np.random.uniform(low=0.05, high=0.4),
                   np.random.uniform(low=0.05, high=0.2))
results_vp = web_interface.vp_similarity_search(query, 1)
results_vp
results_isax = web_interface.isax_similarity_search(query)
results_isax<jupyter_output><empty_output><jupyter_text>Finally, let's store our iSAX tree representation.<jupyter_code>results_tree = web_interface.isax_tree()
print(results_tree)<jupyter_output>root
--->['10', '00', '01', '11']: 4 ['ts-12', 'ts-36', 'ts-37', 'ts-41']
--->['01', '00', '10', '11']: 3 ['ts-14', 'ts-17', 'ts-28']
--->['11', '10', '00', '01']: 3 ['ts-26', 'ts-31', 'ts-5']
--->['11', '10', '01', '00']: 2 ['ts-20', 'ts-42']
--->['11', '00', '00', '11']: 1 ['ts-0']
--->['00', '01', '11', '11']: 5 ['ts-13', 'ts-19', 'ts-22', 'ts-3', 'ts-44']
--->['11', '10', '00', '00']: 4 ['ts-10', 'ts-4', 'ts-45', 'ts-49']
--->['11', '01', '00', '01']: 3 ['ts-18', 'ts-2', 'ts-47']
--->['00', '00', '10', '11']: 3 ['ts-25', 'ts-8', 'ts-9']
--->['10', '00', '10', '11']: 3 ['ts-11', 'ts-16', 'ts-23']
--->['10', '00', '00', '11']: 1 ['ts-24']
--->['00', '01', '10', '11']: 2 ['ts-15', 'ts-27']
--->['11', '11', '01', '00']: 3 ['ts-1', 'ts-29', 'ts-33']
--->['11', '11', '00', '00']: 1 ['ts-40']
--->['01', '00', '01', '11']: 2 ['ts-38', 'ts-43']
--->['11', '01', '00', '10']: 2 ['ts-32', 'ts-46']
--->['00', '01', '10', '10']: 1 ['ts-21']
--->['00', '10', '10', '10']: 2 ['ts-39', 'ts-48']
--->[...]<jupyter_text>Terminate and Reload Database Now that we know that everything is loaded, let's close the database and re-open it.<jupyter_code>os.kill(server.pid, signal.SIGINT)
time.sleep(5)  # give it time to terminate
os.kill(webserver.pid, signal.SIGINT)
time.sleep(5)  # give it time to terminate
web_interface = None
server = subprocess.Popen(['python', '../go_server_persistent.py',
                           '--ts_length', str(ts_length), '--data_dir', data_dir, '--db_name', db_name])
time.sleep(5)  # give it time to load fully
webserver = subprocess.Popen(['python', '../go_webserver.py'])
time.sleep(5)  # give it time to load fully
web_interface = WebInterface()<jupyter_output><empty_output><jupyter_text>Inspect DataLet's repeat the previous tests to check whether our persistence architecture worked.<jupyter_code># select all database entries; all metadata fields
results = web_interface.select(fields=[])

# we have the right number of database entries
assert len(results) == num_ts

# we have all the right primary keys
assert sorted(results.keys()) == ts_keys
# check that all the time series and metadata matches
for k in tsdict:
    results = web_interface.select(fields=['ts'], md={'pk': k})
    assert results[k]['ts'] == tsdict[k]
    results = web_interface.select(fields=[], md={'pk': k})
    for field in metadict[k]:
        assert metadict[k][field] == results[k][field]
# check that the vantage points match
print('Vantage points selected:', vpkeys)
print('Vantage points in database:',
      web_interface.select(fields=None, md={'vp': True}, additional={'sort_by': '+pk'}).keys())
# check that isax tree has fully reloaded
print(web_interface.isax_tree())
# compare vantage point search results
results_vp == web_interface.vp_similarity_search(query, 1)
# compare isax search results
results_isax == web_interface.isax_similarity_search(query)
# check that the trigger is still there by loading new data

# create test time series
_, test = tsmaker(np.random.uniform(low=0.0, high=1.0),
                  np.random.uniform(low=0.05, high=0.4),
                  np.random.uniform(low=0.05, high=0.2))

# insert test time series
web_interface.insert_ts('test', test)

# check that mean and standard deviation have been calculated
print(web_interface.select(fields=['mean', 'std'], md={'pk': 'test'}))

# remove test time series
web_interface.delete_ts('test');<jupyter_output>OrderedDict([('test', OrderedDict([('mean', 1.003503441810608), ('std', 0.725745439529419)]))])<jupyter_text>We have successfully reloaded all of the database components from disk!<jupyter_code># terminate processes before exiting
os.kill(server.pid, signal.SIGINT)
time.sleep(5)  # give it time to terminate
web_interface = None
webserver.terminate()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.161579361953525, 'fraction_numerical': 0.03554549035053171, 'mean_word_length': 4.514115092290988, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '508194', 'license': 'MIT', 'n_tokens_mistral': 3620, 'n_tokens_neox': 3397, 'n_words': 1024, 'path': 'docs/persistence_demo.ipynb', 'repo_name': 'Mynti207/TimeDB', 'size': '10156'}