<commit_before>#!/usr/bin/env python3

import numpy as np
import os
import pandas as pd
import sys
import warnings

from datetime import datetime as dt
from datetime import timedelta as tdelta

from astropy.table import Table
from astropy.time import Time

from astroplan import Observer
from astropy.coordinates import EarthLocation

from pocs.utils.config import load_config

import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt
from matplotlib.dates import DateFormatter
from matplotlib.dates import HourLocator
from matplotlib.dates import MinuteLocator
from matplotlib.ticker import FormatStrFormatter
from matplotlib.ticker import MultipleLocator
plt.ioff()
plt.style.use('classic')


def label_pos(lim, pos=0.85):
    return lim[0] + pos * (lim[1] - lim[0])


class WeatherPlotter(object):

    """ Plot weather information for a given time span """

    def __init__(self, date_string=None, data_file=None, *args, **kwargs):
        super(WeatherPlotter, self).__init__()
        self.args = args
        self.kwargs = kwargs

        config = load_config(config_files=['peas'])
        self.cfg = config['weather']['plot']
        location_cfg = config.get('location', None)

        self.thresholds = config['weather'].get('aag_cloud', None)

        if not date_string:
            self.today = True
            self.date = dt.utcnow()
            self.date_string = self.date.strftime('%Y%m%dUT')
            self.start = self.date - tdelta(1, 0)
            self.end = self.date
            self.lhstart = self.date - tdelta(0, 60 * 60)
            self.lhend = self.date + tdelta(0, 5 * 60)

        else:
            self.today = False
            self.date = dt.strptime('{} 23:59:59'.format(date_string),
                                    '%Y%m%dUT %H:%M:%S')
            self.date_string = date_string
            self.start = dt(self.date.year, self.date.month, self.date.day, 0, 0, 0, 0)
            self.end = dt(self.date.year, self.date.month, self.date.day, 23, 59, 59, 0)
        print('Creating weather plotter for {}'.format(self.date_string))

        self.twilights = self.get_twilights(location_cfg)

        self.table = self.get_table_data(data_file)

        if self.table is None:
            warnings.warn("No data")
            sys.exit(0)

        self.time = pd.to_datetime(self.table['date'])
        first = self.time[0].isoformat()
        last = self.time[-1].isoformat()
        print('  Retrieved {} entries between {} and {}'.format(
              len(self.table), first, last))

        if self.today:
            self.current_values = self.table[-1]
        else:
            self.current_values = None

    def make_plot(self, output_file=None):
        # -------------------------------------------------------------------------
        # Plot a day's weather
        # -------------------------------------------------------------------------
        print('  Setting up plot for time range: {} to {}'.format(
              self.start.isoformat(), self.end.isoformat()))
        if self.today:
            print('  Will generate last hour plot for time range: {} to {}'.format(
                self.lhstart.isoformat(), self.lhend.isoformat()))
        self.dpi = self.kwargs.get('dpi', 72)
        self.fig = plt.figure(figsize=(20, 12), dpi=self.dpi)
#         self.axes = plt.gca()
        self.hours = HourLocator(byhour=range(24), interval=1)
        self.hours_fmt = DateFormatter('%H')
        self.mins = MinuteLocator(range(0, 60, 15))
        self.mins_fmt = DateFormatter('%H:%M')
        self.plot_positions = [([0.000, 0.835, 0.700, 0.170], [0.720, 0.835, 0.280, 0.170]),
                               ([0.000, 0.635, 0.700, 0.170], [0.720, 0.635, 0.280, 0.170]),
                               ([0.000, 0.450, 0.700, 0.170], [0.720, 0.450, 0.280, 0.170]),
                               ([0.000, 0.265, 0.700, 0.170], [0.720, 0.265, 0.280, 0.170]),
                               ([0.000, 0.185, 0.700, 0.065], [0.720, 0.185, 0.280, 0.065]),
                               ([0.000, 0.000, 0.700, 0.170], [0.720, 0.000, 0.280, 0.170]),
                               ]
        self.plot_ambient_vs_time()
        self.plot_cloudiness_vs_time()
        self.plot_windspeed_vs_time()
        self.plot_rain_freq_vs_time()
        self.plot_safety_vs_time()
        self.plot_pwm_vs_time()
        self.save_plot(plot_filename=output_file)

    def get_table_data(self, data_file):
        """ Get the table data

        If a `data_file` (csv) is passed, read from that, otherwise use mongo

        """
        table = None

        col_names = ('ambient_temp_C', 'sky_temp_C', 'sky_condition',
                     'wind_speed_KPH', 'wind_condition',
                     'gust_condition', 'rain_frequency',
                     'rain_condition', 'safe', 'pwm_value',
                     'rain_sensor_temp_C', 'date')

        col_dtypes = ('f4', 'f4', 'U15',
                      'f4', 'U15',
                      'U15', 'f4',
                      'U15', bool, 'f4',
                      'f4', 'O')

        if data_file is not None:
            table = Table.from_pandas(pd.read_csv(data_file, parse_dates=True))
        else:
            # -------------------------------------------------------------------------
            # Grab data from Mongo
            # -------------------------------------------------------------------------
            import pymongo
            from pocs.utils.database import PanDB

            print('  Retrieving data from Mongo database')
            db = PanDB()
            entries = [x for x in db.weather.find(
                {'date': {'$gt': self.start, '$lt': self.end}}).sort([
                    ('date', pymongo.ASCENDING)])]

            table = Table(names=col_names, dtype=col_dtypes)

            for entry in entries:<commit_msg>Handle the weather data when PanDB is used with type of `file`. (#773)<commit_after>#!/usr/bin/env python3

import numpy as np
import os
import pandas as pd
import sys
import warnings

from datetime import datetime as dt
from datetime import timedelta as tdelta

from astropy.table import Table
from astropy.time import Time

from astroplan import Observer
from astropy.coordinates import EarthLocation

from pocs.utils.config import load_config
from pocs.utils import serializers as json_util

import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt
from matplotlib.dates import DateFormatter
from matplotlib.dates import HourLocator
from matplotlib.dates import MinuteLocator
from matplotlib.ticker import FormatStrFormatter
from matplotlib.ticker import MultipleLocator
plt.ioff()
plt.style.use('classic')


def label_pos(lim, pos=0.85):
    return lim[0] + pos * (lim[1] - lim[0])


class WeatherPlotter(object):

    """ Plot weather information for a given time span """

    def __init__(self, date_string=None, data_file=None, *args, **kwargs):
        super(WeatherPlotter, self).__init__()
        self.args = args
        self.kwargs = kwargs

        config = load_config(config_files=['peas'])
        self.cfg = config['weather']['plot']
        location_cfg = config.get('location', None)

        self.thresholds = config['weather'].get('aag_cloud', None)

        if not date_string:
            self.today = True
            self.date = dt.utcnow()
            self.date_string = self.date.strftime('%Y%m%dUT')
            self.start = self.date - tdelta(1, 0)
            self.end = self.date
            self.lhstart = self.date - tdelta(0, 60 * 60)
            self.lhend = self.date + tdelta(0, 5 * 60)

        else:
            self.today = False
            self.date = dt.strptime('{} 23:59:59'.format(date_string),
                                    '%Y%m%dUT %H:%M:%S')
            self.date_string = date_string
            self.start = dt(self.date.year, self.date.month, self.date.day, 0, 0, 0, 0)
            self.end = dt(self.date.year, self.date.month, self.date.day, 23, 59, 59, 0)
        print('Creating weather plotter for {}'.format(self.date_string))

        self.twilights = self.get_twilights(location_cfg)

        self.table = self.get_table_data(data_file)

        if self.table is None:
            warnings.warn("No data")
            sys.exit(0)

        self.time = pd.to_datetime(self.table['date'])
        first = self.time[0].isoformat()
        last = self.time[-1].isoformat()
        print('  Retrieved {} entries between {} and {}'.format(
              len(self.table), first, last))

        if self.today:
            self.current_values = self.table[-1]
        else:
            self.current_values = None

    def make_plot(self, output_file=None):
        # -------------------------------------------------------------------------
        # Plot a day's weather
        # -------------------------------------------------------------------------
        print('  Setting up plot for time range: {} to {}'.format(
              self.start.isoformat(), self.end.isoformat()))
        if self.today:
            print('  Will generate last hour plot for time range: {} to {}'.format(
                self.lhstart.isoformat(), self.lhend.isoformat()))
        self.dpi = self.kwargs.get('dpi', 72)
        self.fig = plt.figure(figsize=(20, 12), dpi=self.dpi)
#         self.axes = plt.gca()
        self.hours = HourLocator(byhour=range(24), interval=1)
        self.hours_fmt = DateFormatter('%H')
        self.mins = MinuteLocator(range(0, 60, 15))
        self.mins_fmt = DateFormatter('%H:%M')
        self.plot_positions = [([0.000, 0.835, 0.700, 0.170], [0.720, 0.835, 0.280, 0.170]),
                               ([0.000, 0.635, 0.700, 0.170], [0.720, 0.635, 0.280, 0.170]),
                               ([0.000, 0.450, 0.700, 0.170], [0.720, 0.450, 0.280, 0.170]),
                               ([0.000, 0.265, 0.700, 0.170], [0.720, 0.265, 0.280, 0.170]),
                               ([0.000, 0.185, 0.700, 0.065], [0.720, 0.185, 0.280, 0.065]),
                               ([0.000, 0.000, 0.700, 0.170], [0.720, 0.000, 0.280, 0.170]),
                               ]
        self.plot_ambient_vs_time()
        self.plot_cloudiness_vs_time()
        self.plot_windspeed_vs_time()
        self.plot_rain_freq_vs_time()
        self.plot_safety_vs_time()
        self.plot_pwm_vs_time()
        self.save_plot(plot_filename=output_file)

    def get_table_data(self, data_file):
        """ Get the table data

        If a `data_file` (csv) is passed, read from that, otherwise use mongo

        """
        table = None

        col_names = ('ambient_temp_C', 'sky_temp_C', 'sky_condition',
                     'wind_speed_KPH', 'wind_condition',
                     'gust_condition', 'rain_frequency',
                     'rain_condition', 'safe', 'pwm_value',
                     'rain_sensor_temp_C', 'date')

        col_dtypes = ('f4', 'f4', 'U15',
                      'f4', 'U15',
                      'U15', 'f4',
                      'U15', bool, 'f4',
                      'f4', 'O')

        if data_file is not None:
            if data_file.endswith('.json'):
                weather_entries = list()
                with open(data_file) as df:
                    for entry in df:
                        rec0 = json_util.loads(entry)
                        weather_entries.append({**rec0['data']})

                table = pd.DataFrame(weather_entries)
                # Fix bad dtype conversion
                table.rain_sensor_temp_C = pd.to_numeric(table.rain_sensor_temp_C)
                table = Table.from_pandas(table)
            else:
                table = Table.from_pandas(pd.read_csv(data_file, parse_dates=True))
        else:
            # -------------------------------------------------------------------------
            # Grab data from Mongo
            # -------------------------------------------------------------------------
            import pymongo
            from pocs.utils.database import PanDB

            print('  Retrieving data from Mongo database')
            db = PanDB()
            entries = [x for x in db.weather.find(
                {'date': {'$gt': self.start, '$lt': self.end}}).sort([
                    ('date', pymongo.ASCENDING)])]

            table = Table(names=col_names, dtype=col_dtypes)

            for entry in entries:
{'fraction_non_alphanumeric': 0.1700274237780287, 'fraction_numerical': 0.04218422326181642, 'mean_word_length': 1.678548282566429, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1272151', 'n_tokens_mistral': 4172, 'n_tokens_neox': 3675, 'n_words': 869}