<commit_before>from pathlib import Path
import configparser
import logging
import pymongo
from datetime import datetime, timedelta
import numpy as np

from bokeh.io import curdoc
from bokeh.plotting import figure
from bokeh.resources import CDN
from bokeh.embed import components
from bokeh.layouts import column
from bokeh.models import DatetimeTickFormatter, NumeralTickFormatter, DatetimeTicker

from iqmon.webpage import mongo_query, get_twilights, overplot_twilights

log = logging.getLogger('FlaskLogger')


##-------------------------------------------------------------------------
## Function: generate_weather_plot
##-------------------------------------------------------------------------
def generate_weather_plot(telescope, date=None, plot_ndays=1, span_hours=24):
    log.info(f"Querying weather limits")
    query_result = mongo_query('weather_limits', {})
    weather_limits = query_result[0]

    cfg_path = Path(__file__).parent.parent / 'configs' / 'pipeline.cfg'
    cfg = configparser.ConfigParser()
    cfg.read(cfg_path)

    if date is None:
        end = datetime.utcnow()
        start = end - timedelta(days=plot_ndays)
    else:
        start = datetime.strptime(date, '%Y%m%dUT')
        end = start + timedelta(days=plot_ndays)

    markersize = 2
    currentweather = {}

    ##-------------------------------------------------------------------------
    ## Temperature Plot
    log.info('Build temperature plot')
    plot_temperature = figure(width=900, height=100, x_axis_type="datetime",
                              y_range=(25,95),
                              x_range=(end - timedelta(hours=span_hours), end),
                              )

    plot_values = cfg['Weather'].get('plot_temperature').split(',')
    query_dict = {'date': {'$gt': start, '$lt': end}}
    for plot_value in plot_values:
        collection, name = plot_value.split(':')
        log.debug(f'  Querying mongo collection {collection}')
        query_result = mongo_query(collection, query_dict)
        plot_vals = np.array([(d['date'], d[name]) for d in query_result])
        log.debug(f'  Got {len(plot_vals)} entries')
        if len(plot_vals) > 0:
            plot_temperature.circle(plot_vals[:,0],
                                   plot_vals[:,1]*1.8+32,
                                   size=markersize, color="blue", alpha=0.8)
        if date is None and 'temperature' not in currentweather.keys():
            currentweather['date'] = plot_vals[-1][0]
            currentweather['temp'] = plot_vals[-1][1]
    plot_temperature.yaxis.axis_label = 'Temp (F)'
    plot_temperature.yaxis.formatter = NumeralTickFormatter(format="0,0")
    plot_temperature.yaxis.ticker = [30, 50, 70, 90]
    plot_temperature.xaxis.visible = False

    ##-------------------------------------------------------------------------
    ## Cloudiness Plot
    log.info('Build cloudiness plot')
    plot_cloudiness = figure(width=900, height=100, x_axis_type="datetime",
                             y_range=(-50,10),
                             x_range=(end - timedelta(hours=span_hours), end),
                             )
    plot_values = cfg['Weather'].get('plot_cloudiness').split(',')
    query_dict = {'date': {'$gt': start, '$lt': end}}
    for plot_value in plot_values:
        collection, name = plot_value.split(':')
        log.debug(f'  Querying mongo collection {collection}')
        query_result = mongo_query(collection, query_dict)
        plot_vals = np.array([(d['date'], d[name]) for d in query_result])
        log.debug(f'  Got {len(plot_vals)} entries')
        if len(plot_vals) > 0:
            where_vcloudy = np.where(plot_vals[:,1] >= weather_limits['cloudy'])
            plot_cloudiness.circle(plot_vals[where_vcloudy][:,0],
                                   plot_vals[where_vcloudy][:,1],
                                   size=markersize, color="red", alpha=0.8)

            where_cloudy = np.where((plot_vals[:,1] < weather_limits['cloudy'])\
                                    & (plot_vals[:,1] <= weather_limits['clear']))
            plot_cloudiness.circle(plot_vals[where_cloudy][:,0],
                                   plot_vals[where_cloudy][:,1],
                                   size=markersize, color="orange", alpha=0.8)

            where_clear = np.where(plot_vals[:,1] < weather_limits['clear'])
            plot_cloudiness.circle(plot_vals[where_clear][:,0],
                                   plot_vals[where_clear][:,1],
                                   size=markersize, color="green", alpha=0.8)
        if date is None and 'clouds' not in currentweather.keys():
            currentweather['clouds'] = plot_vals[-1][1]
    plot_cloudiness.yaxis.axis_label = 'Cloudiness (C)'
    plot_cloudiness.yaxis.formatter = NumeralTickFormatter(format="0,0")
    plot_cloudiness.xaxis.visible = False

    ##-------------------------------------------------------------------------
    ## Wind Plot
    log.info('Build wind plot')
    plot_wind_speed = figure(width=900, height=100, x_axis_type="datetime",
                             y_range=(-3,85),
                             x_range=(end - timedelta(hours=span_hours), end),
                             )
    plot_values = cfg['Weather'].get('plot_wind_speed').split(',')
    query_dict = {'date': {'$gt': start, '$lt': end}}
    for plot_value in plot_values:
        collection, name = plot_value.split(':')
        log.debug(f'  Querying mongo collection {collection}')
        query_result = mongo_query(collection, query_dict)
        plot_vals = np.array([(d['date'], d[name]) for d in query_result])
        log.debug(f'  Got {len(plot_vals)} entries')
        if len(plot_vals) > 0:
            where_vwindy = np.where(plot_vals[:,1] >= weather_limits['windy'])
            plot_wind_speed.circle(plot_vals[where_vwindy][:,0],
                                   plot_vals[where_vwindy][:,1],
                                   size=markersize, color="red", alpha=0.8)

            where_windy = np.where((plot_vals[:,1] < weather_limits['windy'])\
                                    & (plot_vals[:,1] <= weather_limits['calm']))
            plot_wind_speed.circle(plot_vals[where_windy][:,0],
                                   plot_vals[where_windy][:,1],
                                   size=markersize, color="orange", alpha=0.8)

            where_calm = np.where(plot_vals[:,1] < weather_limits['calm'])
            plot_wind_speed.circle(plot_vals[where_calm][:,0],
                                   plot_vals[where_calm][:,1],
                                   size=markersize, color="green", alpha=0.8)
        if date is None and 'wind' not in currentweather.keys():
            currentweather['wind'] = plot_vals[-1][1]
    plot_wind_speed.yaxis.axis_label = 'Wind (kph)'
    plot_wind_speed.yaxis.formatter = NumeralTickFormatter(format="0,0")
    plot_wind_speed.yaxis.ticker = [0, 20, 40, 60, 80]
    plot_wind_speed.xaxis.visible = False

    ##-------------------------------------------------------------------------
    ## Rain Plot
    log.info('Build rain plot')
    plot_rain = figure(width=900, height=60, x_axis_type="datetime",
                       y_range=(500,2800),
                       x_range=(end - timedelta(hours=span_hours), end),
                       )
    plot_values = cfg['Weather'].get('plot_rain').split(',')
    query_dict = {'date': {'$gt': start, '$lt': end}}
    for plot_value in plot_values:
        collection, name = plot_value.split(':')
        log.debug(f'  Querying mongo collection {collection}')
        query_result = mongo_query(collection, query_dict)
        plot_vals = np.array([(d['date'], d[name]) for d in query_result])
        log.debug(f'  Got {len(plot_vals)} entries')
        if len(plot_vals) > 0:
            where_dry = np.where(plot_vals[:,1] >= weather_limits['dry'])
            plot_rain.circle(plot_vals[where_dry][:,0],
                             plot_vals[where_dry][:,1],
                             size=markersize, color="green", alpha=0.8)

            where_wet = np.where((plot_vals[:,1] < weather_limits['dry'])\
                                 & (plot_vals[:,1] <= weather_limits['wet']))
            plot_rain.circle(plot_vals[where_wet][:,0],
                             plot_vals[where_wet][:,1],
                             size=markersize, color="orange", alpha=0.8)

            where_rain = np.where(plot_vals[:,1] < weather_limits['wet'])
            plot_rain.circle(plot_vals[where_rain][:,0],
                             plot_vals[where_rain][:,1],
                             size=markersize, color="red", alpha=0.8)
        if date is None and 'rain' not in currentweather.keys():
            currentweather['rain'] = plot_vals[-1][1]
    plot_rain.yaxis.axis_label = 'Rain'
    plot_rain.yaxis.formatter = NumeralTickFormatter(format="0.0a")
    plot_rain.xaxis.visible = False

    ##-------------------------------------------------------------------------
    ## Safe Plot
    log.info('Build safe plot')
    plot_safe = figure(width=900, height=50, x_axis_type="datetime",
                       y_range=(-0.2,1.2),
                       x_range=(end - timedelta(hours=span_hours), end),
                       )<commit_msg>Fix bug<commit_after>from pathlib import Path
import configparser
import logging
import pymongo
from datetime import datetime, timedelta
import numpy as np

from bokeh.io import curdoc
from bokeh.plotting import figure
from bokeh.resources import CDN
from bokeh.embed import components
from bokeh.layouts import column
from bokeh.models import DatetimeTickFormatter, NumeralTickFormatter, DatetimeTicker

from iqmon.webpage import mongo_query, get_twilights, overplot_twilights

log = logging.getLogger('FlaskLogger')


##-------------------------------------------------------------------------
## Function: generate_weather_plot
##-------------------------------------------------------------------------
def generate_weather_plot(telescope, date=None, plot_ndays=1, span_hours=24):
    log.info(f"Querying weather limits")
    query_result = mongo_query('weather_limits', {})
    weather_limits = query_result[0]

    cfg_path = Path(__file__).parent.parent / 'configs' / 'pipeline.cfg'
    cfg = configparser.ConfigParser()
    cfg.read(cfg_path)

    if date is None:
        end = datetime.utcnow()
        start = end - timedelta(days=plot_ndays)
    else:
        start = datetime.strptime(date, '%Y%m%dUT')
        end = start + timedelta(days=plot_ndays)

    markersize = 2
    currentweather = {}

    ##-------------------------------------------------------------------------
    ## Temperature Plot
    log.info('Build temperature plot')
    plot_temperature = figure(width=900, height=100, x_axis_type="datetime",
                              y_range=(25,95),
                              x_range=(end - timedelta(hours=span_hours), end),
                              )

    plot_values = cfg['Weather'].get('plot_temperature').split(',')
    query_dict = {'date': {'$gt': start, '$lt': end}}
    for plot_value in plot_values:
        collection, name = plot_value.split(':')
        log.debug(f'  Querying mongo collection {collection}')
        query_result = mongo_query(collection, query_dict)
        plot_vals = np.array([(d['date'], d[name]) for d in query_result])
        log.debug(f'  Got {len(plot_vals)} entries')
        if len(plot_vals) > 0:
            plot_temperature.circle(plot_vals[:,0],
                                   plot_vals[:,1]*1.8+32,
                                   size=markersize, color="blue", alpha=0.8)
        if date is None and 'temperature' not in currentweather.keys():
            currentweather['date'] = plot_vals[-1][0]
            currentweather['temp'] = plot_vals[-1][1]
    plot_temperature.yaxis.axis_label = 'Temp (F)'
    plot_temperature.yaxis.formatter = NumeralTickFormatter(format="0,0")
    plot_temperature.yaxis.ticker = [30, 50, 70, 90]
    plot_temperature.xaxis.visible = False

    ##-------------------------------------------------------------------------
    ## Cloudiness Plot
    log.info('Build cloudiness plot')
    plot_cloudiness = figure(width=900, height=100, x_axis_type="datetime",
                             y_range=(-50,10),
                             x_range=(end - timedelta(hours=span_hours), end),
                             )
    plot_values = cfg['Weather'].get('plot_cloudiness').split(',')
    query_dict = {'date': {'$gt': start, '$lt': end}}
    for plot_value in plot_values:
        collection, name = plot_value.split(':')
        log.debug(f'  Querying mongo collection {collection}')
        query_result = mongo_query(collection, query_dict)
        plot_vals = np.array([(d['date'], d[name]) for d in query_result])
        log.debug(f'  Got {len(plot_vals)} entries')
        if len(plot_vals) > 0:
            where_vcloudy = np.where(plot_vals[:,1] >= weather_limits['cloudy'])
            plot_cloudiness.circle(plot_vals[where_vcloudy][:,0],
                                   plot_vals[where_vcloudy][:,1],
                                   size=markersize, color="red", alpha=0.8)

            where_cloudy = np.where((plot_vals[:,1] < weather_limits['cloudy'])\
                                    & (plot_vals[:,1] >= weather_limits['clear']))
            plot_cloudiness.circle(plot_vals[where_cloudy][:,0],
                                   plot_vals[where_cloudy][:,1],
                                   size=markersize, color="orange", alpha=0.8)

            where_clear = np.where(plot_vals[:,1] < weather_limits['clear'])
            plot_cloudiness.circle(plot_vals[where_clear][:,0],
                                   plot_vals[where_clear][:,1],
                                   size=markersize, color="green", alpha=0.8)
        if date is None and 'clouds' not in currentweather.keys():
            currentweather['clouds'] = plot_vals[-1][1]
    plot_cloudiness.yaxis.axis_label = 'Cloudiness (C)'
    plot_cloudiness.yaxis.formatter = NumeralTickFormatter(format="0,0")
    plot_cloudiness.xaxis.visible = False

    ##-------------------------------------------------------------------------
    ## Wind Plot
    log.info('Build wind plot')
    plot_wind_speed = figure(width=900, height=100, x_axis_type="datetime",
                             y_range=(-3,85),
                             x_range=(end - timedelta(hours=span_hours), end),
                             )
    plot_values = cfg['Weather'].get('plot_wind_speed').split(',')
    query_dict = {'date': {'$gt': start, '$lt': end}}
    for plot_value in plot_values:
        collection, name = plot_value.split(':')
        log.debug(f'  Querying mongo collection {collection}')
        query_result = mongo_query(collection, query_dict)
        plot_vals = np.array([(d['date'], d[name]) for d in query_result])
        log.debug(f'  Got {len(plot_vals)} entries')
        if len(plot_vals) > 0:
            where_vwindy = np.where(plot_vals[:,1] >= weather_limits['windy'])
            plot_wind_speed.circle(plot_vals[where_vwindy][:,0],
                                   plot_vals[where_vwindy][:,1],
                                   size=markersize, color="red", alpha=0.8)

            where_windy = np.where((plot_vals[:,1] < weather_limits['windy'])\
                                    & (plot_vals[:,1] >= weather_limits['calm']))
            plot_wind_speed.circle(plot_vals[where_windy][:,0],
                                   plot_vals[where_windy][:,1],
                                   size=markersize, color="orange", alpha=0.8)

            where_calm = np.where(plot_vals[:,1] < weather_limits['calm'])
            plot_wind_speed.circle(plot_vals[where_calm][:,0],
                                   plot_vals[where_calm][:,1],
                                   size=markersize, color="green", alpha=0.8)
        if date is None and 'wind' not in currentweather.keys():
            currentweather['wind'] = plot_vals[-1][1]
    plot_wind_speed.yaxis.axis_label = 'Wind (kph)'
    plot_wind_speed.yaxis.formatter = NumeralTickFormatter(format="0,0")
    plot_wind_speed.yaxis.ticker = [0, 20, 40, 60, 80]
    plot_wind_speed.xaxis.visible = False

    ##-------------------------------------------------------------------------
    ## Rain Plot
    log.info('Build rain plot')
    plot_rain = figure(width=900, height=60, x_axis_type="datetime",
                       y_range=(500,2800),
                       x_range=(end - timedelta(hours=span_hours), end),
                       )
    plot_values = cfg['Weather'].get('plot_rain').split(',')
    query_dict = {'date': {'$gt': start, '$lt': end}}
    for plot_value in plot_values:
        collection, name = plot_value.split(':')
        log.debug(f'  Querying mongo collection {collection}')
        query_result = mongo_query(collection, query_dict)
        plot_vals = np.array([(d['date'], d[name]) for d in query_result])
        log.debug(f'  Got {len(plot_vals)} entries')
        if len(plot_vals) > 0:
            where_dry = np.where(plot_vals[:,1] >= weather_limits['dry'])
            plot_rain.circle(plot_vals[where_dry][:,0],
                             plot_vals[where_dry][:,1],
                             size=markersize, color="green", alpha=0.8)

            where_wet = np.where((plot_vals[:,1] < weather_limits['dry'])\
                                 & (plot_vals[:,1] >= weather_limits['wet']))
            plot_rain.circle(plot_vals[where_wet][:,0],
                             plot_vals[where_wet][:,1],
                             size=markersize, color="orange", alpha=0.8)

            where_rain = np.where(plot_vals[:,1] < weather_limits['wet'])
            plot_rain.circle(plot_vals[where_rain][:,0],
                             plot_vals[where_rain][:,1],
                             size=markersize, color="red", alpha=0.8)
        if date is None and 'rain' not in currentweather.keys():
            currentweather['rain'] = plot_vals[-1][1]
    plot_rain.yaxis.axis_label = 'Rain'
    plot_rain.yaxis.formatter = NumeralTickFormatter(format="0.0a")
    plot_rain.xaxis.visible = False

    ##-------------------------------------------------------------------------
    ## Safe Plot
    log.info('Build safe plot')
    plot_safe = figure(width=900, height=50, x_axis_type="datetime",
                       y_range=(-0.2,1.2),
                       x_range=(end - timedelta(hours=span_hours), end),
                       )
{'fraction_non_alphanumeric': 0.18272031064610075, 'fraction_numerical': 0.016179484413763347, 'mean_word_length': 1.9046052631578947, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7531402', 'n_tokens_mistral': 5628, 'n_tokens_neox': 5334, 'n_words': 947}