<commit_before>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from collections import deque
import time as ttime
import datetime

import pytz
from nose.tools import assert_equal, assert_raises, raises, assert_true

import metadatastore.commands as mdsc
from metadatastore.utils.testing import mds_setup, mds_teardown
from metadatastore.examples.sample_data import temperature_ramp
import uuid

# some useful globals
run_start_uid = None
document_insertion_time = None


# ### Nose setup/teardown methods #############################################


def teardown():
    pass
    mds_teardown()


def setup():
    mds_setup()
    global run_start_uid, document_insertion_time
    document_insertion_time = ttime.time()
    temperature_ramp.run()

    run_start_uid = mdsc.insert_run_start(scan_id=3022013,
                                        beamline_id='testbed',
                                        owner='tester',
                                        group='awesome-devs',
                                        project='Nikea',
                                        time=document_insertion_time,
                                        uid=str(uuid.uuid4()))

# ### Helper functions ########################################################


def setup_syn(custom=None):
    if custom is None:
        custom = {}
    data_keys = {k: {'source': k,
                     'dtype': 'number',
                     'shape': None} for k in '<KEY>'
                 }
    scan_id = 1

    # Create a BeginRunEvent that serves as entry point for a run
    rs = mdsc.insert_run_start(scan_id=scan_id, beamline_id='testing',
                              time=ttime.time(),
                              custom=custom, uid=str(uuid.uuid4()))

    # Create an EventDescriptor that indicates the data
    # keys and serves as header for set of Event(s)
    e_desc = mdsc.insert_descriptor(data_keys=data_keys,
                                    time=ttime.time(),
                                    run_start=rs, uid=str(uuid.uuid4()))
    return rs, e_desc, data_keys


def syn_data(data_keys, count):
    all_data = deque()
    for seq_num in range(count):
        data = {k: float(seq_num) for k in data_keys}
        timestamps = {k: ttime.time() for k in data_keys}

        _time = ttime.time()
        uid = str(uuid.uuid4())
        all_data.append({'data': data, 'timestamps': timestamps,
                         'seq_num': seq_num, 'time': _time,
                         'uid': uid})
    return all_data


# ### Testing metadatastore insertion functionality ###########################


@raises(KeyError)
def check_for_id(document):
    """Make sure that our documents do not have an id field

    Parameters
    ----------
    document : metadatastore.document.Document
        A sanitized mongoengine document
    """
    document['id']


def test_event_descriptor_insertion():
    # format some data keys for insertion
    data_keys = {'some_value': {'source': 'PV:pv1',
                                'shape': [1, 2],
                                'dtype': 'array'},
                 'some_other_val': {'source': 'PV:pv2',
                                    'shape': [],
                                    'dtype': 'number'},
                 'data_key3': {'source': 'PV:pv1',
                               'shape': [],
                               'dtype': 'number',
                               'external': 'FS:foobar'}}
    time = ttime.time()
    # test insert
    ev_desc_uid = mdsc.insert_descriptor(run_start_uid, data_keys, time,
                                         str(uuid.uuid4()))
    ev_desc_mds, = mdsc.find_descriptors(uid=ev_desc_uid)
    # make sure the sanitized event descriptor has no uid
    check_for_id(ev_desc_mds)

    # make sure the event descriptor is pointing to the correct run start
    referenced_run_start = ev_desc_mds['run_start']
    assert_equal(referenced_run_start.uid, run_start_uid)
    assert_equal(ev_desc_mds['time'], time)

    for k in data_keys:
        for ik in data_keys[k]:
            assert_equal(ev_desc_mds.data_keys[k][ik],
                         data_keys[k][ik])


def test_insert_run_start():
    time = ttime.time()
    beamline_id = 'sample_beamline'
    scan_id = 42
    custom = {'foo': 'bar', 'baz': 42,
              'aardvark': ['ants', 3.14]}
    run_start_uid = mdsc.insert_run_start(
        time, beamline_id=beamline_id,
        scan_id=scan_id, custom=custom, uid=str(uuid.uuid4()))

    run_start_mds, = mdsc.find_run_starts(uid=run_start_uid)

    names = ['time', 'beamline_id', 'scan_id'] + list(custom.keys())
    values = [time, beamline_id, scan_id] + list(custom.values())

    for name, val in zip(names, values):
        assert_equal(getattr(run_start_mds, name), val)

    # make sure the metadatstore document raises properly
    check_for_id(run_start_mds)


def test_run_stop_insertion():
    """Test, uh, the insertion of run stop documents
    """
    run_start_uid = mdsc.insert_run_start(
        time=ttime.time(), beamline_id='sample_beamline', scan_id=42,
        uid=str(uuid.uuid4()))
    time = ttime.time()
    exit_status = 'success'
    reason = 'uh, because this is testing and it better be a success?'
    # insert the document
    run_stop_uid = mdsc.insert_run_stop(run_start_uid, time,
                                       exit_status=exit_status,
                                       reason=reason, uid=str(uuid.uuid4()))

    # get the sanitized run_stop document from metadatastore
    run_stop, = mdsc.find_run_stops(uid=run_stop_uid)

    # make sure it does not have an 'id' field
    check_for_id(run_stop)
    # make sure the run stop is pointing to the correct run start
    referenced_run_start = run_stop['run_start']
    assert_equal(referenced_run_start.uid, run_start_uid)

    # check the remaining fields
    comparisons = {'time': time,
                   'exit_status': exit_status,
                   'reason': reason,
                   'uid': run_stop_uid}
    for attr, known_value in comparisons.items():
        assert_equal(known_value, getattr(run_stop, attr))


def test_find_events_smoke():

    num = 50
    rs, e_desc, data_keys = setup_syn()
    all_data = syn_data(data_keys, num)

    mdsc.bulk_insert_events(e_desc, all_data, validate=False)
    mdsc.insert_run_stop(rs, ttime.time(), uid=str(uuid.uuid4()))
    mdsc.clear_process_cache()

    next(mdsc.find_events())


@raises(ValueError)
def test_bad_bulk_insert_event_data():

    num = 50
    rs, e_desc, data_keys = setup_syn()
    all_data = syn_data(data_keys, num)

    # remove one of the keys from the event data
    del all_data[-1]['data']['E']
    mdsc.bulk_insert_events(e_desc, all_data, validate=True)


@raises(ValueError)
def test_bad_bulk_insert_event_timestamp():
    """Test what happens when one event is missing a timestamp for one key"""
    num = 50
    rs, e_desc, data_keys = setup_syn()
    all_data = syn_data(data_keys, num)
    # remove one of the keys from the event timestamps
    del all_data[1]['timestamps']['F']
    mdsc.bulk_insert_events(e_desc, all_data, validate=True)


@raises(mdsc.NoEventDescriptors)
def test_no_evdesc():

    run_start_uid = mdsc.insert_run_start(scan_id=42,
                                        beamline_id='testbed',
                                        owner='tester',
                                        group='awesome-devs',
                                        project='Nikea',
                                        time=document_insertion_time,
                                        uid=str(uuid.uuid4()))

    mdsc.descriptors_by_start(run_start_uid)


# ### Testing metadatastore find functionality ################################
def _find_helper(func, kw):
    func(**kw)


def test_find_funcs_for_smoke():
    """ Exercise documented kwargs in the find_* functions
    """
    rs, = mdsc.find_run_starts(uid=run_start_uid)
    test_dict = {
        mdsc.find_run_starts: [
            {'limit': 5},
            {'start_time': ttime.time()},
            {'start_time': '2015'},
            {'start_time': '2015-03-30'},
            {'start_time': '2015-03-30 03:00:00'},
            {'start_time': datetime.datetime.now()},
            {'stop_time': ttime.time()},
            {'start_time': ttime.time() - 1, 'stop_time': ttime.time()},
            {'beamline_id': 'csx'},
            {'project': 'world-domination'},
            {'owner': 'drdrake'},
            {'scan_id': 1},
            {'uid': run_start_uid}],
        mdsc.find_run_stops: [
            {'start_time': ttime.time()},
            {'stop_time': ttime.time()},
            {'start_time': ttime.time()-1, 'stop_time': ttime.time()},
            {'reason': 'whimsy'},
            {'exit_status': 'success'},
            {'uid': 'foo'}],
        mdsc.find_descriptors: [
            {'run_start': rs},
            {'run_start_uid': rs.uid},
            {'start_time': ttime.time()},
            {'stop_time': ttime.time()},
            {'start_time': ttime.time() - 1, 'stop_time': ttime.time()},
            {'uid': 'foo'}],
        mdsc.find_run_stops: [<commit_msg>TST: Do a better job with the find_funcs_for_smoke<commit_after>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from collections import deque
import time as ttime
import datetime

import pytz
from nose.tools import assert_equal, assert_raises, raises, assert_true

import metadatastore.commands as mdsc
from metadatastore.utils.testing import mds_setup, mds_teardown
from metadatastore.examples.sample_data import temperature_ramp
import uuid

# some useful globals
run_start_uid = None
document_insertion_time = None


# ### Nose setup/teardown methods #############################################


def teardown():
    pass
    mds_teardown()


def setup():
    mds_setup()
    global run_start_uid, document_insertion_time
    document_insertion_time = ttime.time()
    temperature_ramp.run()

    run_start_uid = mdsc.insert_run_start(scan_id=3022013,
                                        beamline_id='testbed',
                                        owner='tester',
                                        group='awesome-devs',
                                        project='Nikea',
                                        time=document_insertion_time,
                                        uid=str(uuid.uuid4()))

# ### Helper functions ########################################################


def setup_syn(custom=None):
    if custom is None:
        custom = {}
    data_keys = {k: {'source': k,
                     'dtype': 'number',
                     'shape': None} for k in '<KEY>'
                 }
    scan_id = 1

    # Create a BeginRunEvent that serves as entry point for a run
    rs = mdsc.insert_run_start(scan_id=scan_id, beamline_id='testing',
                              time=ttime.time(),
                              custom=custom, uid=str(uuid.uuid4()))

    # Create an EventDescriptor that indicates the data
    # keys and serves as header for set of Event(s)
    e_desc = mdsc.insert_descriptor(data_keys=data_keys,
                                    time=ttime.time(),
                                    run_start=rs, uid=str(uuid.uuid4()))
    return rs, e_desc, data_keys


def syn_data(data_keys, count):
    all_data = deque()
    for seq_num in range(count):
        data = {k: float(seq_num) for k in data_keys}
        timestamps = {k: ttime.time() for k in data_keys}

        _time = ttime.time()
        uid = str(uuid.uuid4())
        all_data.append({'data': data, 'timestamps': timestamps,
                         'seq_num': seq_num, 'time': _time,
                         'uid': uid})
    return all_data


# ### Testing metadatastore insertion functionality ###########################


@raises(KeyError)
def check_for_id(document):
    """Make sure that our documents do not have an id field

    Parameters
    ----------
    document : metadatastore.document.Document
        A sanitized mongoengine document
    """
    document['id']


def test_event_descriptor_insertion():
    # format some data keys for insertion
    data_keys = {'some_value': {'source': 'PV:pv1',
                                'shape': [1, 2],
                                'dtype': 'array'},
                 'some_other_val': {'source': 'PV:pv2',
                                    'shape': [],
                                    'dtype': 'number'},
                 'data_key3': {'source': 'PV:pv1',
                               'shape': [],
                               'dtype': 'number',
                               'external': 'FS:foobar'}}
    time = ttime.time()
    # test insert
    ev_desc_uid = mdsc.insert_descriptor(run_start_uid, data_keys, time,
                                         str(uuid.uuid4()))
    ev_desc_mds, = mdsc.find_descriptors(uid=ev_desc_uid)
    # make sure the sanitized event descriptor has no uid
    check_for_id(ev_desc_mds)

    # make sure the event descriptor is pointing to the correct run start
    referenced_run_start = ev_desc_mds['run_start']
    assert_equal(referenced_run_start.uid, run_start_uid)
    assert_equal(ev_desc_mds['time'], time)

    for k in data_keys:
        for ik in data_keys[k]:
            assert_equal(ev_desc_mds.data_keys[k][ik],
                         data_keys[k][ik])


def test_insert_run_start():
    time = ttime.time()
    beamline_id = 'sample_beamline'
    scan_id = 42
    custom = {'foo': 'bar', 'baz': 42,
              'aardvark': ['ants', 3.14]}
    run_start_uid = mdsc.insert_run_start(
        time, beamline_id=beamline_id,
        scan_id=scan_id, custom=custom, uid=str(uuid.uuid4()))

    run_start_mds, = mdsc.find_run_starts(uid=run_start_uid)

    names = ['time', 'beamline_id', 'scan_id'] + list(custom.keys())
    values = [time, beamline_id, scan_id] + list(custom.values())

    for name, val in zip(names, values):
        assert_equal(getattr(run_start_mds, name), val)

    # make sure the metadatstore document raises properly
    check_for_id(run_start_mds)


def test_run_stop_insertion():
    """Test, uh, the insertion of run stop documents
    """
    run_start_uid = mdsc.insert_run_start(
        time=ttime.time(), beamline_id='sample_beamline', scan_id=42,
        uid=str(uuid.uuid4()))
    time = ttime.time()
    exit_status = 'success'
    reason = 'uh, because this is testing and it better be a success?'
    # insert the document
    run_stop_uid = mdsc.insert_run_stop(run_start_uid, time,
                                       exit_status=exit_status,
                                       reason=reason, uid=str(uuid.uuid4()))

    # get the sanitized run_stop document from metadatastore
    run_stop, = mdsc.find_run_stops(uid=run_stop_uid)

    # make sure it does not have an 'id' field
    check_for_id(run_stop)
    # make sure the run stop is pointing to the correct run start
    referenced_run_start = run_stop['run_start']
    assert_equal(referenced_run_start.uid, run_start_uid)

    # check the remaining fields
    comparisons = {'time': time,
                   'exit_status': exit_status,
                   'reason': reason,
                   'uid': run_stop_uid}
    for attr, known_value in comparisons.items():
        assert_equal(known_value, getattr(run_stop, attr))


def test_find_events_smoke():

    num = 50
    rs, e_desc, data_keys = setup_syn()
    all_data = syn_data(data_keys, num)

    mdsc.bulk_insert_events(e_desc, all_data, validate=False)
    mdsc.insert_run_stop(rs, ttime.time(), uid=str(uuid.uuid4()))
    mdsc.clear_process_cache()

    next(mdsc.find_events())


@raises(ValueError)
def test_bad_bulk_insert_event_data():

    num = 50
    rs, e_desc, data_keys = setup_syn()
    all_data = syn_data(data_keys, num)

    # remove one of the keys from the event data
    del all_data[-1]['data']['E']
    mdsc.bulk_insert_events(e_desc, all_data, validate=True)


@raises(ValueError)
def test_bad_bulk_insert_event_timestamp():
    """Test what happens when one event is missing a timestamp for one key"""
    num = 50
    rs, e_desc, data_keys = setup_syn()
    all_data = syn_data(data_keys, num)
    # remove one of the keys from the event timestamps
    del all_data[1]['timestamps']['F']
    mdsc.bulk_insert_events(e_desc, all_data, validate=True)


@raises(mdsc.NoEventDescriptors)
def test_no_evdesc():

    run_start_uid = mdsc.insert_run_start(scan_id=42,
                                        beamline_id='testbed',
                                        owner='tester',
                                        group='awesome-devs',
                                        project='Nikea',
                                        time=document_insertion_time,
                                        uid=str(uuid.uuid4()))

    mdsc.descriptors_by_start(run_start_uid)


# ### Testing metadatastore find functionality ################################
def _find_helper(func, kw):
    # dereference the generator...
    list(func(**kw))


def test_find_funcs_for_smoke():
    rs, = mdsc.find_run_starts(uid=run_start_uid)
    test_dict = {
        mdsc.find_run_starts: [
            {'limit': 5},
            {'start_time': ttime.time()},
            {'start_time': '2015'},
            {'start_time': '2015-03-30'},
            {'start_time': '2015-03-30 03:00:00'},
            {'start_time': datetime.datetime.now()},
            {'stop_time': ttime.time()},
            {'start_time': ttime.time() - 1, 'stop_time': ttime.time()},
            {'beamline_id': 'csx'},
            {'project': 'world-domination'},
            {'owner': 'drdrake'},
            {'scan_id': 1},
            {'uid': run_start_uid}],
        mdsc.find_run_stops: [
            {'start_time': ttime.time()},
            {'stop_time': ttime.time()},
            {'start_time': ttime.time()-1, 'stop_time': ttime.time()},
            {'reason': 'whimsy'},
            {'exit_status': 'success'},
            {'uid': 'foo'}],
        mdsc.find_descriptors: [
            {'run_start': rs},
            {'run_start': rs.uid},
            {'start_time': ttime.time()},
            {'stop_time': ttime.time()},
            {'start_time': ttime.time() - 1, 'stop_time': ttime.time()},
            {'uid': 'foo'}],
        mdsc.find_run_stops: [
{'fraction_non_alphanumeric': 0.12992360621986238, 'fraction_numerical': 0.008018637915154142, 'mean_word_length': 2.1660377358490566, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1767978', 'n_tokens_mistral': 5737, 'n_tokens_neox': 5515, 'n_words': 1223}