<commit_before>#!/usr/bin/env python
'''
uframe endpoints
'''
# base
from flask import jsonify, request, current_app, make_response, Response
from ooiservices.app import cache
from ooiservices.app.uframe import uframe as api
from ooiservices.app.models import PlatformDeployment
from ooiservices.app.main.routes import get_display_name_by_rd, get_long_display_name_by_rd, get_platform_display_name_by_rd
from ooiservices.app.main.authentication import auth
from ooiservices.app.main.errors import internal_server_error
# data imports
from ooiservices.app.uframe.data import get_data, get_simple_data, find_parameter_ids, get_multistream_data
from ooiservices.app.uframe.plotting import generate_plot
from ooiservices.app.uframe.assetController import get_events_by_ref_des

from urllib import urlencode
from datetime import datetime
from dateutil.parser import parse as parse_date
import requests
import json
import numpy as np
import pytz
from contextlib import closing
import time
import urllib2
from copy import deepcopy
from operator import itemgetter


__author__ = '<NAME>'


requests.adapters.DEFAULT_RETRIES = 2
CACHE_TIMEOUT = 86400


def dfs_streams():

    uframe_url, timeout, timeout_read = get_uframe_info()

    TOC = uframe_url+'/toc'

    streams = []

    try:
        payload = requests.get(TOC, timeout=(timeout, timeout_read))
    except requests.exceptions.ConnectionError as e:
        error = "Error: Cannot connect to uframe.  %s" % e
        print error
        return make_response(error, 500)

    toc = payload.json()

    for instrument in toc:

        parameters_dict = parameters_in_instrument(instrument)
        streams = data_streams_in_instrument(instrument, parameters_dict, streams)

    return streams

def parameters_in_instrument(instrument):
    parameters_dict = {}
    parameter_list = []

    stream_parameters = []
    stream_variable_type = []
    stream_units = []
    stream_variables_shape = []

    for parameter in instrument['instrument_parameters']:
        if parameter['shape'].lower() in ['scalar', 'function']:
            if parameter['stream'] not in parameters_dict.iterkeys():

                parameters_dict[parameter['stream']] = []
                parameters_dict[parameter['stream']+'_variable_type'] = []
                parameters_dict[parameter['stream']+'_units'] = []
                parameters_dict[parameter['stream']+'_variables_shape'] = []
                parameters_dict[parameter['stream']+'_pdId'] = []

            parameters_dict[parameter['stream']].append(parameter['particleKey'])
            parameters_dict[parameter['stream']+'_variable_type'].append(parameter['type'].lower())
            parameters_dict[parameter['stream']+'_units'].append(parameter['units'])
            parameters_dict[parameter['stream']+'_variables_shape'].append(parameter['shape'].lower())
            parameters_dict[parameter['stream']+'_pdId'].append(parameter['pdId'].lower())

    return parameters_dict

def data_streams_in_instrument(instrument, parameters_dict, streams):
    for data_stream in instrument['streams']:
       stream = (
           instrument['platform_code'],
           instrument['mooring_code'],
           instrument['instrument_code'],
           data_stream['method'],
           data_stream['stream'],
           instrument['reference_designator'],
           data_stream['beginTime'],
           data_stream['endTime'],
           parameters_dict[data_stream['stream']],
           parameters_dict[data_stream['stream']+'_variable_type'],
           parameters_dict[data_stream['stream']+'_units'],
           parameters_dict[data_stream['stream']+'_variables_shape'],
           parameters_dict[data_stream['stream']+'_pdId']
           )
       #current_app.logger.info("GET %s", each['reference_designator'])
       streams.append(stream)

    return streams

def split_stream_name(ui_stream_name):
    '''
    Splits the hypenated reference designator and stream type into a tuple of
    (mooring, platform, instrument, stream_type, stream)
    '''

    print ui_stream_name
    mooring, platform, instrument = ui_stream_name.split('-', 2)
    instrument, stream_type, stream = instrument.split('_', 2)
    return (mooring, platform, instrument, stream_type, stream)


def combine_stream_name(mooring, platform, instrument, stream_type, stream):
    first_part = '-'.join([mooring, platform, instrument])
    all_of_it = '_'.join([first_part, stream_type, stream])
    return all_of_it


def iso_to_timestamp(iso8601):
    dt = parse_date(iso8601)
    t = (dt - datetime(1970, 1, 1, tzinfo=pytz.utc)).total_seconds()
    return t

def dict_from_stream(mooring, platform, instrument, stream_type, stream, reference_designator, beginTime, endTime, variables, variable_type, units, variables_shape, parameter_id):
    HOST = str(current_app.config['HOST'])
    PORT = str(current_app.config['PORT'])
    SERVICE_LOCATION = 'http://'+HOST+":"+PORT

    ref = mooring + "-" + platform + "-" + instrument
    stream_name = '_'.join([stream_type, stream])
    ref = '-'.join([mooring, platform, instrument])

    data_dict = {}
    data_dict['start'] = beginTime
    data_dict['end'] = endTime
    data_dict['reference_designator'] = reference_designator
    data_dict['stream_name'] = stream_name
    data_dict['variables'] = []
    data_dict['variable_types'] = {}
    data_dict['units'] = {}
    data_dict['variables_shape'] = {}
    data_dict['display_name'] = get_display_name_by_rd(ref)
    data_dict['long_display_name'] = get_long_display_name_by_rd(ref)
    data_dict['platform_name'] = get_platform_display_name_by_rd(ref)
    data_dict['download'] = {
                             "csv":"/".join(['api/uframe/get_csv', stream_name, ref]),
                             "json":"/".join(['api/uframe/get_json', stream_name, ref]),
                             "netcdf":"/".join(['api/uframe/get_netcdf', stream_name, ref]),
                             "profile":"/".join(['api/uframe/get_profiles', stream_name, ref])
                            }
    data_dict['variables'] = variables
    data_dict['variable_type'] = variable_type
    data_dict['units'] = units
    data_dict['variables_shape'] = variables_shape
    data_dict['parameter_id'] = parameter_id


    return data_dict


@api.route('/stream')
<EMAIL>
def streams_list():
    '''
    Accepts stream_name or reference_designator as a URL argument
    '''

    if request.args.get('stream_name'):
        dict_from_stream(request.args.get('stream_name'))

    cached = cache.get('stream_list')
    if cached:
        retval = cached
    else:
        streams = dfs_streams()
        if type(streams) is Response and  streams.status_code != 200:
            return make_response("Error in streams, please make sure uframe connection is open.", streams.status_code)

        retval = []
        for stream in streams:
            try:
                data_dict = dict_from_stream(*stream)
            except Exception as e:
                current_app.logger.exception('\n**** (3) exception: ' + e.message)
                continue
            if request.args.get('reference_designator'):
                if request.args.get('reference_designator') != data_dict['reference_designator']:
                    continue
            retval.append(data_dict)
        cache.set('stream_list', retval, timeout=CACHE_TIMEOUT)

    try:
        is_reverse = False
        if request.args.get('sort') and request.args.get('sort') != "":
            sort_by = request.args.get('sort')
            if request.args.get('order') and request.args.get('order') != "":
                order = request.args.get('order')
                if order == 'reverse':
                    is_reverse = True
        else:
            sort_by = 'end'
        retval = sorted(retval, key=itemgetter(sort_by), reverse=is_reverse)
    except (TypeError, KeyError) as e:
        raise

    if request.args.get('min') == 'True':
        for obj in retval:
            try:
                del obj['parameter_id']
                del obj['units']
                del obj['variable_type']
                del obj['variable_types']
                del obj['download']
                del obj['variables']
                del obj['variables_shape']
            except KeyError:
                raise

    if request.args.get('search') and request.args.get('search') != "":
        return_list = []
        ven_set = []
        search_term = str(request.args.get('search')).split()
        search_set = set(search_term)
        for subset in search_set:
            if len(return_list) > 0:
                if len(ven_set) > 0:
                    ven_set = deepcopy(ven_subset)
                else:
                    ven_set = deepcopy(return_list)
                ven_subset = []
                for item in ven_set:
                    if subset.lower() in str(item['long_display_name']).lower():
                        ven_subset.append(item)
                    elif subset.lower() in str(item['reference_designator']).lower():
                        ven_subset.append(item)
                    elif subset.lower() in str(item['variables']).lower():
                        ven_subset.append(item)
                    elif subset.lower() in str(item['stream_name']).lower():
                        ven_subset.append(item)
                retval = ven_subset
            else:
                for item in retval:
                    if subset.lower() in str(item['long_display_name']).lower():
                        return_list.append(item)
                    elif subset.lower() in str(item['reference_designator']).lower():
                        return_list.append(item)
                    elif subset.lower() in str(item['variables']).lower():
                        return_list.append(item)
                    elif subset.lower() in str(item['stream_name']).lower():
                        return_list.append(item)
                retval = return_list

    if request.args.get('startAt'):
        start_at = int(request.args.get('startAt'))
        count = int(request.args.get('count'))
        total = int(len(retval))
        retval_slice = retval[start_at:(start_at + count)]
        result = jsonify({"count": count,
                            "total": total,
                            "startAt": start_at,
                            "streams": retval_slice})
        return result

    else:
        return jsonify(streams=retval)


<EMAIL>
@api.route('/get_glider_tracks')
def get_uframe_glider_track():
    '''
    get glider tracks
    '''
    try:
        cached = cache.get('glider_tracks')
        will_reset_cache = False
        if request.args.get('reset') == 'true':
            will_reset_cache = True

        will_reset = request.args.get('reset')
        if cached and not(will_reset_cache):
            data = cached
        else:
            # we will always want the telemetered data, and the engineering stream
            glider_ids = []
            glider_locations = []

            base_url, timeout, timeout_read = get_uframe_info()
            #get the list of mobile assets
            r = requests.get(base_url)
            all_platforms = r.json()

            for p in all_platforms:
                if "MOAS" in p:
                    r_p = requests.get(base_url+"/"+p)
                    try:
                        p_p = r_p.json()
                        for gl in p_p:
                            glider_location = "/"+p+"/"+gl+"/00-ENG000000/"
                            glider_locations.append(base_url+glider_location)
                            glider_name =  glider_location + 'telemetered/glider_eng_telemetered'
                            url = base_url+glider_name
                            glider_ids.append(url)
                    except:
                        print "error:", p, r_p.content

            #params for position and depth info
            params = "?parameters=PD1335,PD1336,PD1276&limit=1000"
            data = []
            print len(glider_ids)," gliders..."
            for i,gl_id in enumerate(glider_ids):
                try:
                    #print glider_locations[i]+'metadata'
                    r_units = requests.get(glider_locations[i]+'metadata')
                    d_units = r_units.json()
                    d_units = d_units['parameters']

                    #create the units
                    glider_depth_units = "m"

                    for row in d_units:
                        if row['particleKey'] == 'm_depth':
                            #override them in case something different
                            glider_depth_units = row['particleKey']

                    data_url = gl_id + params
                    #print data_url
                    r = requests.get(data_url)
                    metadata = r.json()

                    coors = []
                    dt = []
                    depths = []
                    for row in metadata:

                        has_lon   = not np.isnan(row['m_gps_lon'])
                        has_lat   = not np.isnan(row['m_gps_lat'])
                        has_depth = not np.isnan(row['m_depth'])
                        #only add the glider information if deoth is available
                        if has_lat and has_lon and has_depth and (float(row['m_depth']) != -999):
                            #add position
                            coors.append([row['m_gps_lon'], row['m_gps_lat']])
                            dt.append(row['pk']['time'])
                            #add depth
                            depths.append(row['m_depth'])

                    #add glider info to dict
                    data_item = {"name":row['pk']['subsite']+"-"+row['pk']['node'],
                                 "reference_designator": row['pk']['subsite']+"-"+row['pk']['node']+"-"+row['pk']['sensor'],
                                 "type": "LineString",
                                 "coordinates" : coors,
                                 "times": dt,
                                 "units": glider_depth_units,
                                 "depths": depths}
                except:
                    continue
                data.append(data_item)
            if "error" not in data:
                cache.set('glider_tracks', data, timeout=CACHE_TIMEOUT)

        return jsonify({"gliders":data})
    except requests.exceptions.ConnectionError as e:
        error = "Error: Cannot connect to uframe.  %s" % e
        print error
        return make_response(error, 500)

<EMAIL>(timeout=3600)
def get_uframe_streams(mooring, platform, instrument, stream_type):
    '''
    Lists all the streams
    '''
    try:
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = '/'.join([uframe_url, mooring, platform, instrument, stream_type])
        current_app.logger.info("GET %s", url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        return response
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))


<EMAIL>(timeout=3600)
def get_uframe_stream(mooring, platform, instrument, stream):
    '''
    Lists the reference designators for the streams
    '''
    try:
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, stream])
        current_app.logger.info("GET %s", url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        return response
    except Exception as e:
        #return internal_server_error('uframe connection cannot be made.' + str(e.message))
        return _response_internal_server_error()

def get_uframe_toc():
    uframe_url = current_app.config['UFRAME_URL'] + current_app.config['UFRAME_TOC']
    r = requests.get(uframe_url)
    if r.status_code == 200:
        d =  r.json()
        for row in d:
            try:
                # FIX FOR THE WRONG WAY ROUND
                temp1 = row['platform_code']
                temp2 = row['mooring_code']
                row['mooring_code'] = temp1
                row['platform_code'] = temp2
                #

                instrument_display_name = PlatformDeployment._get_display_name(row['reference_designator'])
                split_name = instrument_display_name.split(' - ')
                row['instrument_display_name'] = split_name[-1]
                row['mooring_display_name'] = split_name[0]
                row['platform_display_name'] = split_name[1]
            except:
                row['instrument_display_name'] = ""
                row['platform_display_name'] = ""
                row['mooring_display_name'] = ""
        return d
    else:
        return []

@api.route('/get_structured_toc')
@cache.memoize(timeout=1600)
def get_structured_toc():
    try:
        mooring_list = []
        mooring_key = []

        platform_list = []
        platform_key = []

        instrument_list = []
        instrument_key = []

        data = get_uframe_toc()

        for d in data:
            if d['reference_designator'] not in instrument_key:
                instrument_list.append({'array_code':d['reference_designator'][0:2],
                                        'display_name': d['instrument_display_name'],
                                        'mooring_code': d['mooring_code'],
                                        'platform_code': d['platform_code'],
                                        'instrument_code': d['platform_code'],
                                        'streams':d['streams'],
                                        'instrument_parameters':d['instrument_parameters'],
                                        'reference_designator':d['reference_designator']
                                     })

                instrument_key.append(d['reference_designator'])


            if d['mooring_code'] not in mooring_key:
                mooring_list.append({'array_code':d['reference_designator'][0:2],
                                     'mooring_code':d['mooring_code'],
                                     'platform_code':d['platform_code'],
                                     'display_name':d['mooring_display_name'],
                                     'geo_location':[],
                                     'reference_designator':d['mooring_code']
                                     })

                mooring_key.append(d['mooring_code'])

            if d['mooring_code']+d['platform_code'] not in platform_key:
                platform_list.append({'array_code':d['reference_designator'][0:2],
                                      'platform_code':d['platform_code'],
                                      'mooring_code':d['mooring_code'],
                                      'reference_designator':d['reference_designator'],
                                      'display_name': d['platform_display_name']
                                        })

                platform_key.append(d['mooring_code']+d['platform_code'])

        return jsonify(toc={"moorings":mooring_list,
                            "platforms":platform_list,
                            "instruments":instrument_list
                            })
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))

@api.route('/get_toc')
@cache.memoize(timeout=1600)
def get_toc():
    try:
        data = get_uframe_toc()
        return jsonify(toc=data)
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))

@api.route('/get_instrument_metadata/<string:ref>', methods=['GET'])
<EMAIL>(timeout=3600)
def get_uframe_instrument_metadata(ref):
    '''
    Returns the uFrame metadata response for a given stream
    '''
    try:
        mooring, platform, instrument = ref.split('-', 2)
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, 'metadata'])
        response = requests.get(url, timeout=(timeout, timeout_read))
        if response.status_code == 200:
            data = response.json()
            return jsonify(metadata=data['parameters'])
        return jsonify(metadata={}), 404
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))

@api.route('/get_metadata_parameters/<string:ref>', methods=['GET'])
<EMAIL>(timeout=3600)
def get_uframe_instrument_metadata_parameters(ref):
    '''
    Returns the uFrame metadata parameters for a given stream
    '''
    try:
        mooring, platform, instrument = ref.split('-', 2)
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, 'metadata', 'parameters'])
        #current_app.logger.info("GET %s", url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        return response
    except:
        return _response_internal_server_error()

def _response_internal_server_error(msg=None):
    message = json.dumps('"error" : "uframe connection cannot be made."')
    if msg:
        message = json.dumps(msg)
    response = make_response()
    response.content = message
    response.status_code = 500
    response.headers["Content-Type"] = "application/json"
    return response

@auth.login_required
@api.route('/get_metadata_times/<string:ref>', methods=['GET'])
<EMAIL>(timeout=3600)
def get_uframe_stream_metadata_times(ref):
    '''
    Returns the uFrame time bounds response for a given stream
    '''
    mooring, platform, instrument = ref.split('-', 2)
    try:
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, 'metadata','times'])
        #current_app.logger.info("GET %s", url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        if response.status_code == 200:
            return response
        return jsonify(times={}), 200
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))

<EMAIL>(timeout=3600)
#DEPRECATED
def get_uframe_stream_contents(mooring, platform, instrument, stream_type, stream, start_time, end_time, dpa_flag, provenance='false', annotations='false'):
    """
    Gets the bounded stream contents, start_time and end_time need to be datetime objects; returns Respnse object.
    """
    try:
        if dpa_flag == '0':
            query = '?beginDT=%s&endDT=%s&include_provenance=%s&include_annotations=%s' % (start_time, end_time, provenance, annotations)
        else:
            query = '?beginDT=%s&endDT=%s&include_provenance=%s&include_annotations=%s&execDPA=true' % (start_time, end_time, provenance, annotations)
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, stream_type, stream + query])
        current_app.logger.debug('***** url: ' + url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        if not response:
            raise Exception('No data available from uFrame for this request.')
        if response.status_code != 200:
            raise Exception('(%s) failed to retrieve stream contents from uFrame', response.status_code)
            #pass
        return response
    except Exception as e:
        return internal_server_error('uFrame connection cannot be made. ' + str(e.message))


@auth.login_required
@api.route('/get_multistream/<string:stream1>/<string:stream2>/<string:instrument1>/<string:instrument2>/<string:var1>/<string:var2>', methods=['GET'])
def multistream_api(stream1, stream2, instrument1, instrument2, var1, var2):
    '''
    Service endpoint to get multistream interploated data
    Example request:
        http://localhost:4000/uframe/get_multistream/CP05MOAS-GL340-03-CTDGVM000/CP05MOAS-GL340-02-FLORTM000/telemetered_ctdgv_m_glider_instrument/
        telemetered_flort_m_glider_instrument/sci_water_pressure/sci_flbbcd_chlor_units?startdate=2015-05-07T02:49:22.745Z&enddate=2015-06-28T04:00:41.282Z
    '''
    try:
        resp_data, units1, units2 = get_multistream_data(instrument1, instrument2, stream1, stream2, var1, var2)
    except Exception as err:
        return jsonify(error='%s' % str(err.message)), 400
    return jsonify(data=resp_data, units=[units1, units2])


def get_uframe_multi_stream_contents(stream1_dict, stream2_dict, start_time, end_time):
    '''
    Gets the data from an interpolated multi stream request.

    For details on the UFrame API:
        https://uframe-cm.ooi.rutgers.edu/projects/ooi/wiki/Web_Interface

    Example request:
        http://uframe-test.ooi.rutgers.edu:12576/sensor?r=r1&r=r2&r1.refdes=CP05MOAS-GL340-03-CTDGVM000&
        r2.refdes=CP05MOAS-GL340-02-FLORTM000&r1.method=telemetered&r2.method=telemetered&r1.stream=ctdgv_m_glider_instrument&
        r2.stream=flort_m_glider_instrument&r1.params=PD1527&r2.params=PD1485&limit=1000&startDT=2015-05-07T02:49:22.745Z&endDT=2015-06-28T04:00:41.282Z
    '''

    # Get the parts of the request from the input stream dicts
    refdes1 = stream1_dict['refdes']
    refdes2 = stream2_dict['refdes']

    method1 = stream1_dict['method']
    method2 = stream2_dict['method']

    stream1 = stream1_dict['stream']
    stream2 = stream2_dict['stream']

    params1 = stream1_dict['params']
    params2 = stream2_dict['params']

    limit = current_app.config['DATA_POINTS']

    GA_URL = current_app.config['GOOGLE_ANALYTICS_URL']+'&ec=multistreamdata&ea=%s&el=%s' % ('-'.join([refdes1+stream1, refdes1+stream2]), '-'.join([start_time, end_time]))

    query = ('sensor?r=r1&r=r2&r1.refdes=%s&r2.refdes=%s&r1.method=%s&r2.method=%s'
             '&r1.stream=%s&r2.stream=%s&r1.params=%s&r2.params=%s&limit=%s&startDT=%s&endDT=%s'
             % (refdes1, refdes2, method1, method2, stream1, stream2,
                params1, params2, limit, start_time, end_time))

    url = "/".join([current_app.config['UFRAME_URL'], query])

    current_app.logger.debug("***:" + url)

    _, timeout, timeout_read = get_uframe_info()
    response = requests.get(url, timeout=(timeout, timeout_read))

    return response.text, response.status_code


def get_uframe_plot_contents_chunked(mooring, platform, instrument, stream_type, stream, start_time, end_time, dpa_flag, parameter_ids):
    '''
    Gets the bounded stream contents, start_time and end_time need to be datetime objects
    '''
    try:
        if dpa_flag == '0' and len(parameter_ids)<1:
            query = '?beginDT=%s&endDT=%s&limit=%s' % (start_time, end_time, current_app.config['DATA_POINTS'])
        elif dpa_flag == '1' and len(parameter_ids)<1:<commit_msg>Update the response<commit_after>#!/usr/bin/env python
'''
uframe endpoints
'''
# base
from flask import jsonify, request, current_app, make_response, Response
from ooiservices.app import cache
from ooiservices.app.uframe import uframe as api
from ooiservices.app.models import PlatformDeployment
from ooiservices.app.main.routes import get_display_name_by_rd, get_long_display_name_by_rd, get_platform_display_name_by_rd
from ooiservices.app.main.authentication import auth
from ooiservices.app.main.errors import internal_server_error
# data imports
from ooiservices.app.uframe.data import get_data, get_simple_data, find_parameter_ids, get_multistream_data
from ooiservices.app.uframe.plotting import generate_plot
from ooiservices.app.uframe.assetController import get_events_by_ref_des

from urllib import urlencode
from datetime import datetime
from dateutil.parser import parse as parse_date
import requests
import json
import numpy as np
import pytz
from contextlib import closing
import time
import urllib2
from copy import deepcopy
from operator import itemgetter


__author__ = '<NAME>'


requests.adapters.DEFAULT_RETRIES = 2
CACHE_TIMEOUT = 86400


def dfs_streams():

    uframe_url, timeout, timeout_read = get_uframe_info()

    TOC = uframe_url+'/toc'

    streams = []

    try:
        payload = requests.get(TOC, timeout=(timeout, timeout_read))
    except requests.exceptions.ConnectionError as e:
        error = "Error: Cannot connect to uframe.  %s" % e
        print error
        return make_response(error, 500)

    toc = payload.json()

    for instrument in toc:

        parameters_dict = parameters_in_instrument(instrument)
        streams = data_streams_in_instrument(instrument, parameters_dict, streams)

    return streams

def parameters_in_instrument(instrument):
    parameters_dict = {}
    parameter_list = []

    stream_parameters = []
    stream_variable_type = []
    stream_units = []
    stream_variables_shape = []

    for parameter in instrument['instrument_parameters']:
        if parameter['shape'].lower() in ['scalar', 'function']:
            if parameter['stream'] not in parameters_dict.iterkeys():

                parameters_dict[parameter['stream']] = []
                parameters_dict[parameter['stream']+'_variable_type'] = []
                parameters_dict[parameter['stream']+'_units'] = []
                parameters_dict[parameter['stream']+'_variables_shape'] = []
                parameters_dict[parameter['stream']+'_pdId'] = []

            parameters_dict[parameter['stream']].append(parameter['particleKey'])
            parameters_dict[parameter['stream']+'_variable_type'].append(parameter['type'].lower())
            parameters_dict[parameter['stream']+'_units'].append(parameter['units'])
            parameters_dict[parameter['stream']+'_variables_shape'].append(parameter['shape'].lower())
            parameters_dict[parameter['stream']+'_pdId'].append(parameter['pdId'].lower())

    return parameters_dict

def data_streams_in_instrument(instrument, parameters_dict, streams):
    for data_stream in instrument['streams']:
       stream = (
           instrument['platform_code'],
           instrument['mooring_code'],
           instrument['instrument_code'],
           data_stream['method'],
           data_stream['stream'],
           instrument['reference_designator'],
           data_stream['beginTime'],
           data_stream['endTime'],
           parameters_dict[data_stream['stream']],
           parameters_dict[data_stream['stream']+'_variable_type'],
           parameters_dict[data_stream['stream']+'_units'],
           parameters_dict[data_stream['stream']+'_variables_shape'],
           parameters_dict[data_stream['stream']+'_pdId']
           )
       #current_app.logger.info("GET %s", each['reference_designator'])
       streams.append(stream)

    return streams

def split_stream_name(ui_stream_name):
    '''
    Splits the hypenated reference designator and stream type into a tuple of
    (mooring, platform, instrument, stream_type, stream)
    '''

    print ui_stream_name
    mooring, platform, instrument = ui_stream_name.split('-', 2)
    instrument, stream_type, stream = instrument.split('_', 2)
    return (mooring, platform, instrument, stream_type, stream)


def combine_stream_name(mooring, platform, instrument, stream_type, stream):
    first_part = '-'.join([mooring, platform, instrument])
    all_of_it = '_'.join([first_part, stream_type, stream])
    return all_of_it


def iso_to_timestamp(iso8601):
    dt = parse_date(iso8601)
    t = (dt - datetime(1970, 1, 1, tzinfo=pytz.utc)).total_seconds()
    return t

def dict_from_stream(mooring, platform, instrument, stream_type, stream, reference_designator, beginTime, endTime, variables, variable_type, units, variables_shape, parameter_id):
    HOST = str(current_app.config['HOST'])
    PORT = str(current_app.config['PORT'])
    SERVICE_LOCATION = 'http://'+HOST+":"+PORT

    ref = mooring + "-" + platform + "-" + instrument
    stream_name = '_'.join([stream_type, stream])
    ref = '-'.join([mooring, platform, instrument])

    data_dict = {}
    data_dict['start'] = beginTime
    data_dict['end'] = endTime
    data_dict['reference_designator'] = reference_designator
    data_dict['stream_name'] = stream_name
    data_dict['variables'] = []
    data_dict['variable_types'] = {}
    data_dict['units'] = {}
    data_dict['variables_shape'] = {}
    data_dict['display_name'] = get_display_name_by_rd(ref)
    data_dict['long_display_name'] = get_long_display_name_by_rd(ref)
    data_dict['platform_name'] = get_platform_display_name_by_rd(ref)
    data_dict['download'] = {
                             "csv":"/".join(['api/uframe/get_csv', stream_name, ref]),
                             "json":"/".join(['api/uframe/get_json', stream_name, ref]),
                             "netcdf":"/".join(['api/uframe/get_netcdf', stream_name, ref]),
                             "profile":"/".join(['api/uframe/get_profiles', stream_name, ref])
                            }
    data_dict['variables'] = variables
    data_dict['variable_type'] = variable_type
    data_dict['units'] = units
    data_dict['variables_shape'] = variables_shape
    data_dict['parameter_id'] = parameter_id


    return data_dict


@api.route('/stream')
<EMAIL>
def streams_list():
    '''
    Accepts stream_name or reference_designator as a URL argument
    '''

    if request.args.get('stream_name'):
        dict_from_stream(request.args.get('stream_name'))

    cached = cache.get('stream_list')
    if cached:
        retval = cached
    else:
        streams = dfs_streams()
        if type(streams) is Response and  streams.status_code != 200:
            return make_response("Error in streams, please make sure uframe connection is open.", streams.status_code)

        retval = []
        for stream in streams:
            try:
                data_dict = dict_from_stream(*stream)
            except Exception as e:
                current_app.logger.exception('\n**** (3) exception: ' + e.message)
                continue
            if request.args.get('reference_designator'):
                if request.args.get('reference_designator') != data_dict['reference_designator']:
                    continue
            retval.append(data_dict)
        cache.set('stream_list', retval, timeout=CACHE_TIMEOUT)

    try:
        is_reverse = False
        if request.args.get('sort') and request.args.get('sort') != "":
            sort_by = request.args.get('sort')
            if request.args.get('order') and request.args.get('order') != "":
                order = request.args.get('order')
                if order == 'reverse':
                    is_reverse = True
        else:
            sort_by = 'end'
        retval = sorted(retval, key=itemgetter(sort_by), reverse=is_reverse)
    except (TypeError, KeyError) as e:
        raise

    if request.args.get('min') == 'True':
        for obj in retval:
            try:
                del obj['parameter_id']
                del obj['units']
                del obj['variable_type']
                del obj['variable_types']
                del obj['download']
                del obj['variables']
                del obj['variables_shape']
            except KeyError:
                raise

    if request.args.get('search') and request.args.get('search') != "":
        return_list = []
        ven_set = []
        search_term = str(request.args.get('search')).split()
        search_set = set(search_term)
        for subset in search_set:
            if len(return_list) > 0:
                if len(ven_set) > 0:
                    ven_set = deepcopy(ven_subset)
                else:
                    ven_set = deepcopy(return_list)
                ven_subset = []
                for item in ven_set:
                    if subset.lower() in str(item['long_display_name']).lower():
                        ven_subset.append(item)
                    elif subset.lower() in str(item['reference_designator']).lower():
                        ven_subset.append(item)
                    elif subset.lower() in str(item['variables']).lower():
                        ven_subset.append(item)
                    elif subset.lower() in str(item['stream_name']).lower():
                        ven_subset.append(item)
                retval = ven_subset
            else:
                for item in retval:
                    if subset.lower() in str(item['long_display_name']).lower():
                        return_list.append(item)
                    elif subset.lower() in str(item['reference_designator']).lower():
                        return_list.append(item)
                    elif subset.lower() in str(item['variables']).lower():
                        return_list.append(item)
                    elif subset.lower() in str(item['stream_name']).lower():
                        return_list.append(item)
                retval = return_list

    if request.args.get('startAt'):
        start_at = int(request.args.get('startAt'))
        count = int(request.args.get('count'))
        total = int(len(retval))
        retval_slice = retval[start_at:(start_at + count)]
        result = jsonify({"count": count,
                            "total": total,
                            "startAt": start_at,
                            "streams": retval_slice})
        return result

    else:
        return jsonify(streams=retval)


<EMAIL>.login_required
@api.route('/get_glider_tracks')
def get_uframe_glider_track():
    '''
    get glider tracks
    '''
    try:
        cached = cache.get('glider_tracks')
        will_reset_cache = False
        if request.args.get('reset') == 'true':
            will_reset_cache = True

        will_reset = request.args.get('reset')
        if cached and not(will_reset_cache):
            data = cached
        else:
            # we will always want the telemetered data, and the engineering stream
            glider_ids = []
            glider_locations = []

            base_url, timeout, timeout_read = get_uframe_info()
            #get the list of mobile assets
            r = requests.get(base_url)
            all_platforms = r.json()

            for p in all_platforms:
                if "MOAS" in p:
                    r_p = requests.get(base_url+"/"+p)
                    try:
                        p_p = r_p.json()
                        for gl in p_p:
                            glider_location = "/"+p+"/"+gl+"/00-ENG000000/"
                            glider_locations.append(base_url+glider_location)
                            glider_name =  glider_location + 'telemetered/glider_eng_telemetered'
                            url = base_url+glider_name
                            glider_ids.append(url)
                    except:
                        print "error:", p, r_p.content

            #params for position and depth info
            params = "?parameters=PD1335,PD1336,PD1276&limit=1000"
            data = []
            print len(glider_ids)," gliders..."
            for i,gl_id in enumerate(glider_ids):
                try:
                    #print glider_locations[i]+'metadata'
                    r_units = requests.get(glider_locations[i]+'metadata')
                    d_units = r_units.json()
                    d_units = d_units['parameters']

                    #create the units
                    glider_depth_units = "m"

                    for row in d_units:
                        if row['particleKey'] == 'm_depth':
                            #override them in case something different
                            glider_depth_units = row['particleKey']

                    data_url = gl_id + params
                    #print data_url
                    r = requests.get(data_url)
                    metadata = r.json()

                    coors = []
                    dt = []
                    depths = []
                    for row in metadata:

                        has_lon   = not np.isnan(row['m_gps_lon'])
                        has_lat   = not np.isnan(row['m_gps_lat'])
                        has_depth = not np.isnan(row['m_depth'])
                        #only add the glider information if deoth is available
                        if has_lat and has_lon and has_depth and (float(row['m_depth']) != -999):
                            #add position
                            coors.append([row['m_gps_lon'], row['m_gps_lat']])
                            dt.append(row['pk']['time'])
                            #add depth
                            depths.append(row['m_depth'])

                    #add glider info to dict
                    data_item = {"name":row['pk']['subsite']+"-"+row['pk']['node'],
                                 "reference_designator": row['pk']['subsite']+"-"+row['pk']['node']+"-"+row['pk']['sensor'],
                                 "type": "LineString",
                                 "coordinates" : coors,
                                 "times": dt,
                                 "units": glider_depth_units,
                                 "depths": depths}
                except:
                    continue
                data.append(data_item)
            if "error" not in data:
                cache.set('glider_tracks', data, timeout=CACHE_TIMEOUT)

        return jsonify({"gliders":data})
    except requests.exceptions.ConnectionError as e:
        error = "Error: Cannot connect to uframe.  %s" % e
        print error
        return make_response(error, 500)

<EMAIL>(timeout=3600)
def get_uframe_streams(mooring, platform, instrument, stream_type):
    '''
    Lists all the streams
    '''
    try:
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = '/'.join([uframe_url, mooring, platform, instrument, stream_type])
        current_app.logger.info("GET %s", url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        return response
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))


<EMAIL>(timeout=3600)
def get_uframe_stream(mooring, platform, instrument, stream):
    '''
    Lists the reference designators for the streams
    '''
    try:
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, stream])
        current_app.logger.info("GET %s", url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        return response
    except Exception as e:
        #return internal_server_error('uframe connection cannot be made.' + str(e.message))
        return _response_internal_server_error()

def get_uframe_toc():
    uframe_url = current_app.config['UFRAME_URL'] + current_app.config['UFRAME_TOC']
    r = requests.get(uframe_url)
    if r.status_code == 200:
        d =  r.json()
        for row in d:
            try:
                # FIX FOR THE WRONG WAY ROUND
                temp1 = row['platform_code']
                temp2 = row['mooring_code']
                row['mooring_code'] = temp1
                row['platform_code'] = temp2
                #

                instrument_display_name = PlatformDeployment._get_display_name(row['reference_designator'])
                split_name = instrument_display_name.split(' - ')
                row['instrument_display_name'] = split_name[-1]
                row['mooring_display_name'] = split_name[0]
                row['platform_display_name'] = split_name[1]
            except:
                row['instrument_display_name'] = ""
                row['platform_display_name'] = ""
                row['mooring_display_name'] = ""
        return d
    else:
        return []

@api.route('/get_structured_toc')
@cache.memoize(timeout=1600)
def get_structured_toc():
    try:
        mooring_list = []
        mooring_key = []

        platform_list = []
        platform_key = []

        instrument_list = []
        instrument_key = []

        data = get_uframe_toc()

        for d in data:
            if d['reference_designator'] not in instrument_key:
                instrument_list.append({'array_code':d['reference_designator'][0:2],
                                        'display_name': d['instrument_display_name'],
                                        'mooring_code': d['mooring_code'],
                                        'platform_code': d['platform_code'],
                                        'instrument_code': d['platform_code'],
                                        'streams':d['streams'],
                                        'instrument_parameters':d['instrument_parameters'],
                                        'reference_designator':d['reference_designator']
                                     })

                instrument_key.append(d['reference_designator'])


            if d['mooring_code'] not in mooring_key:
                mooring_list.append({'array_code':d['reference_designator'][0:2],
                                     'mooring_code':d['mooring_code'],
                                     'platform_code':d['platform_code'],
                                     'display_name':d['mooring_display_name'],
                                     'geo_location':[],
                                     'reference_designator':d['mooring_code']
                                     })

                mooring_key.append(d['mooring_code'])

            if d['mooring_code']+d['platform_code'] not in platform_key:
                platform_list.append({'array_code':d['reference_designator'][0:2],
                                      'platform_code':d['platform_code'],
                                      'mooring_code':d['mooring_code'],
                                      'reference_designator':d['reference_designator'],
                                      'display_name': d['platform_display_name']
                                        })

                platform_key.append(d['mooring_code']+d['platform_code'])

        return jsonify(toc={"moorings":mooring_list,
                            "platforms":platform_list,
                            "instruments":instrument_list
                            })
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))

@api.route('/get_toc')
@cache.memoize(timeout=1600)
def get_toc():
    try:
        data = get_uframe_toc()
        return jsonify(toc=data)
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))

@api.route('/get_instrument_metadata/<string:ref>', methods=['GET'])
<EMAIL>(timeout=3600)
def get_uframe_instrument_metadata(ref):
    '''
    Returns the uFrame metadata response for a given stream
    '''
    try:
        mooring, platform, instrument = ref.split('-', 2)
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, 'metadata'])
        response = requests.get(url, timeout=(timeout, timeout_read))
        if response.status_code == 200:
            data = response.json()
            return jsonify(metadata=data['parameters'])
        return jsonify(metadata={}), 404
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))

@api.route('/get_metadata_parameters/<string:ref>', methods=['GET'])
<EMAIL>(timeout=3600)
def get_uframe_instrument_metadata_parameters(ref):
    '''
    Returns the uFrame metadata parameters for a given stream
    '''
    try:
        mooring, platform, instrument = ref.split('-', 2)
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, 'metadata', 'parameters'])
        #current_app.logger.info("GET %s", url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        return response
    except:
        return _response_internal_server_error()

def _response_internal_server_error(msg=None):
    message = json.dumps('"error" : "uframe connection cannot be made."')
    if msg:
        message = json.dumps(msg)
    response = make_response()
    response.content = message
    response.status_code = 500
    response.headers["Content-Type"] = "application/json"
    return response

@auth.login_required
@api.route('/get_metadata_times/<string:ref>', methods=['GET'])
<EMAIL>(timeout=3600)
def get_uframe_stream_metadata_times(ref):
    '''
    Returns the uFrame time bounds response for a given stream
    '''
    mooring, platform, instrument = ref.split('-', 2)
    try:
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, 'metadata','times'])
        #current_app.logger.info("GET %s", url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        if response.status_code == 200:
            return response
        return jsonify(times={}), 200
    except Exception as e:
        return internal_server_error('uframe connection cannot be made.' + str(e.message))

<EMAIL>(timeout=3600)
#DEPRECATED
def get_uframe_stream_contents(mooring, platform, instrument, stream_type, stream, start_time, end_time, dpa_flag, provenance='false', annotations='false'):
    """
    Gets the bounded stream contents, start_time and end_time need to be datetime objects; returns Respnse object.
    """
    try:
        if dpa_flag == '0':
            query = '?beginDT=%s&endDT=%s&include_provenance=%s&include_annotations=%s' % (start_time, end_time, provenance, annotations)
        else:
            query = '?beginDT=%s&endDT=%s&include_provenance=%s&include_annotations=%s&execDPA=true' % (start_time, end_time, provenance, annotations)
        uframe_url, timeout, timeout_read = get_uframe_info()
        url = "/".join([uframe_url, mooring, platform, instrument, stream_type, stream + query])
        current_app.logger.debug('***** url: ' + url)
        response = requests.get(url, timeout=(timeout, timeout_read))
        if not response:
            raise Exception('No data available from uFrame for this request.')
        if response.status_code != 200:
            raise Exception('(%s) failed to retrieve stream contents from uFrame', response.status_code)
            #pass
        return response
    except Exception as e:
        return internal_server_error('uFrame connection cannot be made. ' + str(e.message))


@auth.login_required
@api.route('/get_multistream/<string:stream1>/<string:stream2>/<string:instrument1>/<string:instrument2>/<string:var1>/<string:var2>', methods=['GET'])
def multistream_api(stream1, stream2, instrument1, instrument2, var1, var2):
    '''
    Service endpoint to get multistream interploated data
    Example request:
        http://localhost:4000/uframe/get_multistream/CP05MOAS-GL340-03-CTDGVM000/CP05MOAS-GL340-02-FLORTM000/telemetered_ctdgv_m_glider_instrument/
        telemetered_flort_m_glider_instrument/sci_water_pressure/sci_flbbcd_chlor_units?startdate=2015-05-07T02:49:22.745Z&enddate=2015-06-28T04:00:41.282Z
    '''
    try:
        resp_data, units1, units2 = get_multistream_data(instrument1, instrument2, stream1, stream2, var1, var2)
    except Exception as err:
        return jsonify(error='%s' % str(err.message)), 400

    header1 = '-'.join([stream1, instrument1.replace('_', '-', 1)])
    header2 = '-'.join([stream2, instrument2.replace('_', '-', 1)])

    # Need to reformat the data a bit
    try:
        for ind, data in enumerate(resp_data[header2]):
            resp_data[header1][ind][var2] = data[var2]
    except IndexError:
        return internal_server_error('UFrame array length error')

    return jsonify(data=resp_data[header1], units=[units1, units2])


def get_uframe_multi_stream_contents(stream1_dict, stream2_dict, start_time, end_time):
    '''
    Gets the data from an interpolated multi stream request.

    For details on the UFrame API:
        https://uframe-cm.ooi.rutgers.edu/projects/ooi/wiki/Web_Interface

    Example request:
        http://uframe-test.ooi.rutgers.edu:12576/sensor?r=r1&r=r2&r1.refdes=CP05MOAS-GL340-03-CTDGVM000&
        r2.refdes=CP05MOAS-GL340-02-FLORTM000&r1.method=telemetered&r2.method=telemetered&r1.stream=ctdgv_m_glider_instrument&
        r2.stream=flort_m_glider_instrument&r1.params=PD1527&r2.params=PD1485&limit=1000&startDT=2015-05-07T02:49:22.745Z&endDT=2015-06-28T04:00:41.282Z
    '''

    # Get the parts of the request from the input stream dicts
    refdes1 = stream1_dict['refdes']
    refdes2 = stream2_dict['refdes']

    method1 = stream1_dict['method']
    method2 = stream2_dict['method']

    stream1 = stream1_dict['stream']
    stream2 = stream2_dict['stream']

    params1 = stream1_dict['params']
    params2 = stream2_dict['params']

    limit = current_app.config['DATA_POINTS']

    GA_URL = current_app.config['GOOGLE_ANALYTICS_URL']+'&ec=multistreamdata&ea=%s&el=%s' % ('-'.join([refdes1+stream1, refdes1+stream2]), '-'.join([start_time, end_time]))

    query = ('sensor?r=r1&r=r2&r1.refdes=%s&r2.refdes=%s&r1.method=%s&r2.method=%s'
             '&r1.stream=%s&r2.stream=%s&r1.params=%s&r2.params=%s&limit=%s&startDT=%s&endDT=%s'
             % (refdes1, refdes2, method1, method2, stream1, stream2,
                params1, params2, limit, start_time, end_time))

    url = "/".join([current_app.config['UFRAME_URL'], query])

    current_app.logger.debug("***:" + url)

    _, timeout, timeout_read = get_uframe_info()
    response = requests.get(url, timeout=(timeout, timeout_read))

    return response.json(), response.status_code


def get_uframe_plot_contents_chunked(mooring, platform, instrument, stream_type, stream, start_time, end_time, dpa_flag, parameter_ids):
    '''
    Gets the bounded stream contents, start_time and end_time need to be datetime objects
    '''
    try:
        if dpa_flag == '0' and len(parameter_ids)<1:
            query = '?beginDT=%s&endDT=%s&limit=%s' % (start_time, end_time, current_app.config['DATA_POINTS'])
        elif dpa_flag == '1' and len(parameter_ids)<1:
{'fraction_non_alphanumeric': 0.11743305632502309, 'fraction_numerical': 0.012797783933518006, 'mean_word_length': 2.328068342449757, 'pattern_counts': {'":': 44, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1064575', 'n_tokens_mistral': 16002, 'n_tokens_neox': 15348, 'n_words': 3045}