<commit_before># model.py

# HYSPLITm - HYSPLIT Manager

# The HYSPLIT model is maintained by the NOAA Air Resources Lab. Users of this
# program should properly credit and reference NOAA ARL.
# For more information please visit:
#   http://www.arl.noaa.gov/HYSPLIT_info.php
#   http://www.arl.noaa.gov/disclaimer.php

# See the README for more information.


import sys
import os
import pickle
import shutil
from time import sleep
import datetime as dt
import subprocess
import re
import multiprocessing
import numpy as np


class HySetup(object):
    """Sets up the HYSPLIT model.
    """

    def __init__(self, parent, n_exe=-1):
        """
        If n_exe is -1, will query user for number of instances of HYSPLIT
        to run simultaneously.
        """
        self._parent = parent
        # Setup directories
        self.h_dir = self._parent._h_dir
        self.var_dir = self._parent._var_dir
        self.work_dir = self._parent._work_dir

        # Look for previous pickled setup file
        if os.path.isfile(os.path.join(self.var_dir, 'SETUP')):
            s_file = open(os.path.join(self.var_dir, 'SETUP'), 'r')
            self.met_dir, self.out_dir, self.n_exe, \
            self.auto_dl, self.incl_ld = pickle.load(s_file)
            s_file.close()
            # Overwrite number of executables to run if necessary
            if self.n_exe != n_exe and n_exe > 0:
                self.n_exe = n_exe
        else:
            # Get HYSPLIT I/O directories
            self.met_dir = raw_input(
                'Enter HYSPLIT meteorology files directory: ')
            self.out_dir = raw_input(
                'Enter HYSPLIT output files directory: ')

            # Get number of HYSPLIT instances to run
            if n_exe == -1:
                ncpu = multiprocessing.cpu_count()
                print '''
        There are %s processor cores on this computer.
        Enter the number of simultaneous HYSPLIT executables you want.
                ''' % ncpu
                while True:
                    try:
                        run_mode = input()
                        if type(run_mode).__name__ == 'int' \
                                and 0 < run_mode <= ncpu:
                            break
                        else:
                            print 'Invalid Entry'
                    except (ValueError, SyntaxError, NameError):
                        print 'Invalid Entry'
                if run_mode:
                    self.n_exe = run_mode
                else:
                    self.n_exe = multiprocessing.cpu_count() - 1
            elif n_exe > 0:
                self.n_exe = n_exe
            else:
                self.n_exe = multiprocessing.cpu_count() - 1

            # Get automatic met file download preferenceprint
            # NOTE: haven't fully tested, not included in this version
            if False:
                print '''
        Automatically download missing met files from NOAA ARL server?
        1 for yes, 0 for no:
                '''
                while True:
                    try:
                        auto_dl = input()
                        if type(auto_dl).__name__ == 'int' \
                                and 0 <= auto_dl <= 1:
                            break
                        else:
                            print 'Invalid Entry'
                    except (ValueError, SyntaxError, NameError):
                        print 'Invalid Entry'
                self.auto_dl = bool(auto_dl)
            else:
                self.auto_dl = False

            # Ask if a receptor location directory should be included in output
            print '''
        Save HYSPLIT output files in directories for each receptor?
        1 for yes, 0 for no:
            '''
            while True:
                try:
                    incl_ld = input()
                    if type(incl_ld).__name__ == 'int' \
                            and incl_ld >= 0 and incl_ld <= 1:
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            self.incl_ld = bool(incl_ld)

            # Pickle HYSPLIT Setup file
            s_file = open(os.path.join(self.var_dir, 'SETUP'), 'w')
            pickle.dump([self.met_dir, self.out_dir,
                         self.n_exe, self.auto_dl, self.incl_ld], s_file)
            s_file.close()

        # Ensure there are enough HYSPLIT working directories for each instance
        for i in range(self.n_exe):
            if not (os.path.isdir(os.path.join(
                    self.work_dir, 'h%s' % (i + 1)))):
                shutil.copytree(os.path.join(self.work_dir, 'hdefault'),
                                os.path.join(self.work_dir, 'h%s' % (i + 1)))


class HyBase(object):
    """Inheritable class with methods for using HYSPLIT model and its output.
    """

    def __init__(self, parent):
        self._parent = parent

    def check_date(self, date):
        """Checks if HYSPLIT dates requested are valid dates.
        Date input format (as string): 'YYYYMMDD'
        Returns True for valid, False for invalid date
        """
        try:
            dt.datetime.strptime(date, '%Y%m%d')
            return True
        except ValueError:
            return False

    def run_dir(self, run):
        """Returns proper directory for run with trailing slash.
        """
        if not hasattr(self, 'HyS'):
            self.HyS = self._parent.HyS

        if self.HyS.incl_ld:
            run_dir = os.path.join(self.HyS.out_dir, self.get_var(run)['met'],
                                   run[0:14], '')
        else:
            run_dir = os.path.join(self.HyS.out_dir, self.get_var(run)['met'])
            run_dir = os.path.join(run_dir, '')
        return run_dir

    def get_name(self, lat, lon, met, ver, dur, alt_type, alt, yr, mo, day, hr, non_d=None):
        """Returns the proper name for a HYSPLIT output file.
        If improper inputs, returns False.
        non_d will hold non-default run options if called.
        NOTE: the duration variable, dur, has been updated to four digits. This
        version is therefore not reverse compatible with runs from older
        versions.
        Parameters:
            lat:        Latitude; int or float. Negative for South.
            lon:        Longitude; int or float. Negative for West.
            met:        Met Dataset; string.
                options: 'gdas1', 'gdas0p5', 'edas40', 'rean', 'cmp4'
            ver:        Vertical Motion; int. 0 to 5, 8 now?. See HYSPLIT user's guide.
                        0 is typical and will use the model vertical velocity.
                options: 0:data 1:isob 2:isen 3:dens 4:sigma 5:diverg 6:msl2agl 7:average 8:damped
            dur:        Trajectory Duration (hours); int. Negative for backtrajectory.
            alt_type:   Altitude Type; int.
                options: 0:AGL, 1:MSL
            alt:        Altitude (m): int.
            yr:         Year; int.
            mo:         Month; int.
            day:        Day; int.
            hr:         Hour; int.
            non_d:      Dictionary with non-default HYSPLIT run options. Leave
                        as None to use only default options.
        """
        n = ''
        n = n + ('%05.2f' % abs(lat))[0:2] + '-' + ('%05.2f' % abs(lat))[3:5]
        if lat > 0:
            n = n + 'N_'
        else:
            n = n + 'S_'
        n = n + ('%06.2f' % abs(lon))[0:3] + '-' + ('%06.2f' % abs(lon))[4:6]
        if lon > 0:
            n = n + 'E_'
        else:
            n = n + 'W_'
        n = n + met + '_'
        n = n + str(ver)
        if dur < 0:
            n = n + 'B'
        if dur > 0:
            n = n + 'F'
        n = n + '%04d' % abs(dur)
        if alt_type == 0:
            n = n + 'A'
        if alt_type == 1:
            n = n + 'M'
        n = n + '%05d' % alt
        n = n + '%04d' % yr + '%02d' % mo + '%02d' % day + '%02d' % hr
        # FIXME: should just get key value pairs from non_d and append.
        if non_d is not None:
            non_d_list = ['mdom', 'met', 'mgmin', 'tratio', 'delt', 'tout', 'khmax']
            for i in non_d_list:
                try:
                    n = n + '_' + i + str(non_d[i])
                except (KeyError, TypeError):
                    pass
        return n

    def get_var(self, run):
        """Returns a dictionary of variables with Run instructions.
        Approximate opposite of get_name().
        Returns False on bad Run name.
        Variable dictionary for Run:
            'lat','lon','met','ver','dur','alt_type',
            'alt','yr','mo','day','hr','non_d'
        If non-default options are set, will return a dictionary of
        non-default options.
        in the value for 'non_d'. Otherwise 'non_d' has a value of 'None'.
        NOTE: the duration variable, dur, has been updated to four digits. This
        version is therefore not reverse compatible with runs from older
        versions.
        """
        try:
            run = run.replace('-', '.')
            # "R":   0=Lat, 1=Lon, 2=Met, 3=main run options,
            #       (4+)=non-default options if present
            R = run.rsplit('_')
            num_non_d = len(R) - 4
            lat = float(R[0][0:5])
            if R[0][5] == 'S':
                lat = lat * -1.
            lon = float(R[1][0:6])
            if R[1][6] == 'W':
                lon = lon * -1.
            met = R[2]
            ver = int(R[3][0])
            dur = int(R[3][2:6])
            if R[3][1] == 'B':
                dur = dur * -1
            if R[3][6] == 'A':
                alt_type = 0
            else:
                alt_type = 1
            alt = int(R[3][7:12])
            yr = int(R[3][12:16])
            mo = int(R[3][16:18])
            day = int(R[3][18:20])
            hr = int(R[3][20:22])
            # Non-Default run options
            # FIXME: Not sure what the met_lets is for. Remnant from previous
            # naming scheme, but shouldn't be needed any longer.
            # Actually, this whole section is weird, need to redo and simplify.
            non_d = {}
            re_var_name = re.compile('[a-z]*')
            for i in range(num_non_d):
                tmp = re_var_name.match(R[4 + i])
                var_name = tmp.group()
                val = R[4 + i][tmp.end():]
                if var_name == 'mdom':
                    non_d[var_name] = val
                elif var_name == 'met':
                    vals = {}
                    met_vars = ['tm_pres', 'tm_tpot', 'tm_tamb', 'tm_rain',
                                'tm_mixd', 'tm_relh', 'tm_sphu', 'tm_mixr',
                                'tm_dswf', 'tm_terr']
                    met_lets = ['P', 'O', 'A', 'R', 'D', 'H', 'E', 'W', 'S', 'T']
                    for j in range(10):
                        if val.find(met_lets[j]) == -1:
                            vals[met_vars[j]] = 0
                        else:
                            vals[met_vars[j]] = 1
                    non_d[var_name] = vals
                else:
                    non_d[var_name] = val
            if not non_d:
                # FIXME: Not sure why I put z in here - look into later
                z = None  # analysis:ignore
            return {'lat': lat, 'lon': lon, 'met': met, 'ver': ver, 'dur': dur,
                    'alt_type': alt_type, 'alt': alt, 'yr': yr, 'mo': mo,
                    'day': day, 'hr': hr, 'non_d': non_d}
        except (ValueError, IndexError):
            return False

    def check_run(self, run):
        """Checks if HYSPLIT 'run' is a valid name.
        Boolean return.
        """
        if self.get_var(run) is False:
            return False
        return True

    def check_ex(self, run, mkdir=False):
        """Checks if HYSPLIT output file and directory exists.
        If mkdir is True, will create the directory if needed.
        Returns None if bad run name.
        Returns False if valid name and file doesn't exist.
        Returns True if valid name and file does exist.
        """
        if not self.check_run(run):
            return None
        run_dir = self.run_dir(run)
        if not (os.path.isdir(
                os.path.join(self.HyS.out_dir, self.get_var(run)['met']))) \
                and mkdir:
            os.mkdir(os.path.join(self.HyS.out_dir, self.get_var(run)['met']))
        if not (os.path.isdir(run_dir)):
            os.mkdir(run_dir)
        if os.path.isfile(os.path.join(run_dir, run)):
            return True
        else:
            return False

    def _list_runs(self):
        """Original method for adding runs to a list by user input.
        Returns a list of run names, or False if no valid runs to return (need
        to rerun input).
        NOTE: This is a very old method from an earlier version intended to
        add runs for a specific situation. There are much better ways to do this
        now, but it is left in for the time being as a simple way to add runs
        in interactive mode.
        """
        # -Get needed runs-
        print 'Start HYSPLIT Runs for:'
        print ''
        print 'Number of Locations:'
        while True:
            try:
                num_loc = input()
                if isinstance(num_loc, int) and num_loc >= -1:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        if num_loc == -1:
            return False
        print ''
        print 'Location (Lat, Long to two decimal places as [X.XX,X.XX]):'
        print '  Note that south lat and west long should be negative'
        lat = []
        lon = []
        for i in range(num_loc):
            print 'Location %s:' % (i + 1)
            while True:
                try:
                    lat_lon = input()
                    if isinstance(lat_lon, list) and len(lat_lon) == 2:
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            lat.append(lat_lon[0])
            lon.append(lat_lon[1])
        print ''
        print '0: Gdas1'
        print '1: Gdas0p5'
        print '2: Edas40'
        print '3: Reanalysis'
        print '4: Coamps4'
        print 'Met Dataset:'
        while True:
            try:
                met_int = input()
                if isinstance(met_int, int) and 0 <= met_int < 5:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        met_dict = {0: 'gdas1', 1: 'gdas0p5', 2: 'edas40', 3: 'rean', 4: 'cmp4'}
        if met_int == 4:
            print 'WARNING: Naming of HYSPLIT COAMPS met datasets are not'
            print 'unique as the grid varies based on the COAMPS run.'
            print 'Care should be taken to ensure the met datasets for the'
            print 'proper run are included in the HYSPLIT met files directory.'
        met = met_dict[met_int]
        print ''
        print 'Vertical Motion option:'
        while True:
            try:
                ver = input()
                if isinstance(ver, int) and 0 <= ver <= 5:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Duration hours (negative for backtrajectories):'
        while True:
            try:
                dur = input()
                if isinstance(dur, int):
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Altitude Type, 0:AGL 1:MSL:'
        while True:
            try:
                alt_type = input()
                if isinstance(alt_type, int) and 0 <= alt_type <= 1:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Number of Altitudes:'
        while True:
            try:
                num_alt = input()
                if isinstance(num_alt, int):
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Altitude (meters):'
        alt = []
        for i in range(num_alt):
            print 'Altitude ' + str(i + 1) + ':'
            while True:
                try:
                    tmp = input()
                    if isinstance(tmp, int):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            alt.append(tmp)
        print ''
        print 'Date Range (Start Date, End Date as [YYYY,MM,DD,YYYY,MM,DD]):'
        while True:
            try:
                date_range = input()
                if isinstance(date_range, list) and len(date_range) == 6:
                    if self.check_date(str(date_range[0]) +
                                               str(date_range[1]) +
                                               str(date_range[2])) == True \
                            and self.check_date(str(date_range[3]) +
                                                        str(date_range[4]) +
                                                        str(date_range[5])) == True:
                        break
                    else:
                        print 'Invalid Date'
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        b_yr, b_mo, b_day, e_yr, e_mo, e_day = date_range
        print ''
        print 'Run at Hours each day: ([X,X,X] etc.) or'
        print '-1 for [0,3,6,9,12,15,18,21]:'
        while True:
            try:
                hrs = input()
                if hrs == -1:
                    hrs = [0, 3, 6, 9, 12, 15, 18, 21]
                    break
                else:
                    if isinstance(hrs, list) and len(hrs) <= 24:
                        break
                    else:
                        print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Change any Default options? (0:No 1:Yes):'
        while True:
            try:
                nd = input()
                if type(nd).__name__ == 'int' and 0 <= nd <= 1:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        if nd:
            print 'Enter value for Non-Default run options.'
            print 'Enter "-1" to leave as default.'
            non_d = {}
            while True:
                try:
                    met_ = input('met (\'string\'): ')
                    if isinstance(met, str):
                        break
                    else:
                        if type(met).__name__ == 'int' and met == -1:
                            break
                        else:
                            print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if met_ != -1:
                non_d['met'] = met_
            while True:
                try:
                    mgmin = input('subgrid size (integer): ')
                    if isinstance(mgmin, int) and (mgmin >= 0 or mgmin == -1):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if mgmin != -1:
                non_d['mgmin'] = mgmin
            while True:
                try:
                    tratio = input('Grid cell ratio fraction (float): ')
                    if isinstance(tratio, float) and 0. <= tratio <= 1.:
                        break
                    else:
                        if isinstance(tratio, int) and tratio == -1:
                            break
                        else:
                            print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if tratio != -1:
                non_d['tratio'] = tratio
            if tratio == -1:
                while True:
                    try:
                        delt = input('integration time step (integer): ')
                        if isinstance(delt, int) \
                                and ((0 <= delt <= 60) or delt == -1):
                            break
                        else:
                            print 'Invalid Entry'
                    except (ValueError, SyntaxError, NameError):
                        print 'Invalid Entry'
                if delt != -1:
                    non_d['delt'] = delt
            while True:
                try:
                    tout = input(
                        'Endpoint output frequency in minutes (integer): ')
                    if isinstance(tout, int) and (tout >= 0 or tout == -1):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if tout != -1:
                non_d['tout'] = tout
            while True:
                try:
                    khmax = input(
                        'maximum trajectory duration in hours (integer): ')
                    if isinstance(khmax, int) and (khmax >= 0 or khmax == -1):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if khmax != -1:
                non_d['khmax'] = khmax
                non_d['mhrs'] = khmax
            while True:
                try:
                    mdom = \
                        input('top of model domain in meters AGL (integer): ')
                    if isinstance(mdom, int) and (mdom >= 0 or mdom == -1):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if mdom != -1:
                non_d['mdom'] = mdom
        else:
            non_d = None
        # -Print options and confirm-
        print ''
        print ''
        print 'Current HYSPLIT Run Request'
        print 'Location(s) (Lat(s), Lon(s)): ', lat, lon
        print 'Met Dataset: ', met
        print 'Vertical Motion Option: ', ver
        print 'Trajectory Duration (negative for backtrajectory): ', dur
        if alt_type == 0:
            tmp = ' AGL'
        else:
            tmp = ' MSL'
        print 'Altitude(s): ', alt, tmp
        print 'Date Range: ', date_range
        print 'Hours to Run at: ', hrs
        if nd:
            print 'Non-Default options: ', non_d
        print 'Is this correct? (0:No 1:Yes): '
        while True:
            try:
                correct = input()
                if isinstance(correct, int) and 0 <= correct <= 1:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        if correct == 0:
            return False
        # -Generate each run time-
        # For n Runs (at one time each) of HYSPLIT there is an index which has
        # each time value at Yr[n], Mo[n] etc.
        n = 0  # The number of times HYSPLIT is to be run for each case
        yr = []
        mo = []
        day = []
        hr = []
        now = dt.date(b_yr, b_mo, b_day)
        end = dt.date(e_yr, e_mo, e_day)
        while now <= end:
            for i in hrs:
                yr.append(now.year)
                mo.append(now.month)
                day.append(now.day)
                hr.append(i)
                n += 1
            now = now + dt.timedelta(1)
        # -Save to list and return-
        ret = []
        for i_loc in range(num_loc):
            for i_alt in range(num_alt):
                for i in range(n):
                    ret.append(self.get_name(lat[i_loc], lon[i_loc], met,
                                             ver, dur, alt_type, alt[i_alt],
                                             yr[i], mo[i], day[i], hr[i],
                                             non_d))
        return ret

    def add_queue(self, ret_runset=False):
        """Queries user to add more runs to HYSPLIT queue.
        Parameters:
            ret_runset:     If True, will return a list of all newly added
                            run names.
                            If False, will return None
        """
        runset = []
        while True:
            # User add runs
            rslt = self._list_runs()
            more = 0
            if rslt:
                runset += rslt
            else:
                print 'Warning: No runs added'
            # Check if more runs are needed
            while True:
                try:
                    more = input('Add more runs? (0:No 1:Yes): ')
                    if isinstance(more, int) and 0 <= more <= 1:
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            # Break user add runs loop if done
            if more == 0:
                break
        # Open Queue.txt to append new runs
        os.chdir(self.HyS.h_dir)
        q_file = open(os.path.join(self.HyS.var_dir, 'Queue.txt'), 'a')
        for r in runset:
            q_file.write(r + '\n')
        print '%s HYSPLIT runs added to Queue' % len(runset)
        q_file.close()
        # Return runset if requested
        if ret_runset:
            return runset<commit_msg>Bug Fix<commit_after># model.py

# HYSPLITm - HYSPLIT Manager

# The HYSPLIT model is maintained by the NOAA Air Resources Lab. Users of this
# program should properly credit and reference NOAA ARL.
# For more information please visit:
#   http://www.arl.noaa.gov/HYSPLIT_info.php
#   http://www.arl.noaa.gov/disclaimer.php

# See the README for more information.


import sys
import os
import pickle
import shutil
from time import sleep
import datetime as dt
import subprocess
import re
import multiprocessing
import numpy as np


class HySetup(object):
    """Sets up the HYSPLIT model.
    """

    def __init__(self, parent, n_exe=-1):
        """
        If n_exe is -1, will query user for number of instances of HYSPLIT
        to run simultaneously.
        """
        self._parent = parent
        # Setup directories
        self.h_dir = self._parent._h_dir
        self.var_dir = self._parent._var_dir
        self.work_dir = self._parent._work_dir

        # Look for previous pickled setup file
        if os.path.isfile(os.path.join(self.var_dir, 'SETUP')):
            s_file = open(os.path.join(self.var_dir, 'SETUP'), 'r')
            self.met_dir, self.out_dir, self.n_exe, \
            self.auto_dl, self.incl_ld = pickle.load(s_file)
            s_file.close()
            # Overwrite number of executables to run if necessary
            if self.n_exe != n_exe and n_exe > 0:
                self.n_exe = n_exe
        else:
            # Get HYSPLIT I/O directories
            self.met_dir = raw_input(
                'Enter HYSPLIT meteorology files directory: ')
            self.out_dir = raw_input(
                'Enter HYSPLIT output files directory: ')

            # Get number of HYSPLIT instances to run
            if n_exe == -1:
                ncpu = multiprocessing.cpu_count()
                print '''
        There are %s processor cores on this computer.
        Enter the number of simultaneous HYSPLIT executables you want.
                ''' % ncpu
                while True:
                    try:
                        run_mode = input()
                        if type(run_mode).__name__ == 'int' \
                                and 0 < run_mode <= ncpu:
                            break
                        else:
                            print 'Invalid Entry'
                    except (ValueError, SyntaxError, NameError):
                        print 'Invalid Entry'
                if run_mode:
                    self.n_exe = run_mode
                else:
                    self.n_exe = multiprocessing.cpu_count() - 1
            elif n_exe > 0:
                self.n_exe = n_exe
            else:
                self.n_exe = multiprocessing.cpu_count() - 1

            # Get automatic met file download preferenceprint
            # NOTE: haven't fully tested, not included in this version
            if False:
                print '''
        Automatically download missing met files from NOAA ARL server?
        1 for yes, 0 for no:
                '''
                while True:
                    try:
                        auto_dl = input()
                        if type(auto_dl).__name__ == 'int' \
                                and 0 <= auto_dl <= 1:
                            break
                        else:
                            print 'Invalid Entry'
                    except (ValueError, SyntaxError, NameError):
                        print 'Invalid Entry'
                self.auto_dl = bool(auto_dl)
            else:
                self.auto_dl = False

            # Ask if a receptor location directory should be included in output
            print '''
        Save HYSPLIT output files in directories for each receptor?
        1 for yes, 0 for no:
            '''
            while True:
                try:
                    incl_ld = input()
                    if type(incl_ld).__name__ == 'int' \
                            and incl_ld >= 0 and incl_ld <= 1:
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            self.incl_ld = bool(incl_ld)

            # Pickle HYSPLIT Setup file
            s_file = open(os.path.join(self.var_dir, 'SETUP'), 'w')
            pickle.dump([self.met_dir, self.out_dir,
                         self.n_exe, self.auto_dl, self.incl_ld], s_file)
            s_file.close()

        # Ensure there are enough HYSPLIT working directories for each instance
        for i in range(self.n_exe):
            if not (os.path.isdir(os.path.join(
                    self.work_dir, 'h%s' % (i + 1)))):
                shutil.copytree(os.path.join(self.work_dir, 'hdefault'),
                                os.path.join(self.work_dir, 'h%s' % (i + 1)))


class HyBase(object):
    """Inheritable class with methods for using HYSPLIT model and its output.
    """

    def __init__(self, parent):
        self._parent = parent

    def check_date(self, date):
        """Checks if HYSPLIT dates requested are valid dates.
        Date input format (as string): 'YYYYMMDD'
        Returns True for valid, False for invalid date
        """
        try:
            dt.datetime.strptime(date, '%Y%m%d')
            return True
        except ValueError:
            return False

    def run_dir(self, run):
        """Returns proper directory for run with trailing slash.
        """
        if not hasattr(self, 'HyS'):
            self.HyS = self._parent.HyS

        if self.HyS.incl_ld:
            run_dir = os.path.join(self.HyS.out_dir, self.get_var(run)['met'],
                                   run[0:14], '')
        else:
            run_dir = os.path.join(self.HyS.out_dir, self.get_var(run)['met'])
            run_dir = os.path.join(run_dir, '')
        return run_dir

    def get_name(self, lat, lon, met, ver, dur, alt_type, alt, yr, mo, day, hr, non_d=None):
        """Returns the proper name for a HYSPLIT output file.
        If improper inputs, returns False.
        non_d will hold non-default run options if called.
        NOTE: the duration variable, dur, has been updated to four digits. This
        version is therefore not reverse compatible with runs from older
        versions.
        Parameters:
            lat:        Latitude; int or float. Negative for South.
            lon:        Longitude; int or float. Negative for West.
            met:        Met Dataset; string.
                options: 'gdas1', 'gdas0p5', 'edas40', 'rean', 'cmp4'
            ver:        Vertical Motion; int. 0 to 5, 8 now?. See HYSPLIT user's guide.
                        0 is typical and will use the model vertical velocity.
                options: 0:data 1:isob 2:isen 3:dens 4:sigma 5:diverg 6:msl2agl 7:average 8:damped
            dur:        Trajectory Duration (hours); int. Negative for backtrajectory.
            alt_type:   Altitude Type; int.
                options: 0:AGL, 1:MSL
            alt:        Altitude (m): int.
            yr:         Year; int.
            mo:         Month; int.
            day:        Day; int.
            hr:         Hour; int.
            non_d:      Dictionary with non-default HYSPLIT run options. Leave
                        as None to use only default options.
        """
        n = ''
        n = n + ('%05.2f' % abs(lat))[0:2] + '-' + ('%05.2f' % abs(lat))[3:5]
        if lat > 0:
            n = n + 'N_'
        else:
            n = n + 'S_'
        n = n + ('%06.2f' % abs(lon))[0:3] + '-' + ('%06.2f' % abs(lon))[4:6]
        if lon > 0:
            n = n + 'E_'
        else:
            n = n + 'W_'
        n = n + met + '_'
        n = n + str(ver)
        if dur < 0:
            n = n + 'B'
        if dur > 0:
            n = n + 'F'
        n = n + '%04d' % abs(dur)
        if alt_type == 0:
            n = n + 'A'
        if alt_type == 1:
            n = n + 'M'
        n = n + '%05d' % alt
        n = n + '%04d' % yr + '%02d' % mo + '%02d' % day + '%02d' % hr
        # FIXME: should just get key value pairs from non_d and append.
        if non_d is not None:
            non_d_list = ['mdom', 'met', 'mgmin', 'tratio', 'delt', 'tout', 'khmax']
            for i in non_d_list:
                try:
                    n = n + '_' + i + str(non_d[i])
                except (KeyError, TypeError):
                    pass
        return n

    def get_var(self, run):
        """Returns a dictionary of variables with Run instructions.
        Approximate opposite of get_name().
        Returns False on bad Run name.
        Variable dictionary for Run:
            'lat','lon','met','ver','dur','alt_type',
            'alt','yr','mo','day','hr','non_d'
        If non-default options are set, will return a dictionary of
        non-default options.
        in the value for 'non_d'. Otherwise 'non_d' has a value of 'None'.
        NOTE: the duration variable, dur, has been updated to four digits. This
        version is therefore not reverse compatible with runs from older
        versions.
        """
        try:
            run = run.replace('-', '.')
            # "R":   0=Lat, 1=Lon, 2=Met, 3=main run options,
            #       (4+)=non-default options if present
            R = run.rsplit('_')
            num_non_d = len(R) - 4
            lat = float(R[0][0:5])
            if R[0][5] == 'S':
                lat = lat * -1.
            lon = float(R[1][0:6])
            if R[1][6] == 'W':
                lon = lon * -1.
            met = R[2]
            ver = int(R[3][0])
            dur = int(R[3][2:6])
            if R[3][1] == 'B':
                dur = dur * -1
            if R[3][6] == 'A':
                alt_type = 0
            else:
                alt_type = 1
            alt = int(R[3][7:12])
            yr = int(R[3][12:16])
            mo = int(R[3][16:18])
            day = int(R[3][18:20])
            hr = int(R[3][20:22])
            # Non-Default run options
            # FIXME: Not sure what the met_lets is for. Remnant from previous
            # naming scheme, but shouldn't be needed any longer.
            # Actually, this whole section is weird, need to redo and simplify.
            non_d = {}
            re_var_name = re.compile('[a-z]*')
            for i in range(num_non_d):
                tmp = re_var_name.match(R[4 + i])
                var_name = tmp.group()
                val = R[4 + i][tmp.end():]
                if var_name == 'mdom':
                    non_d[var_name] = val
                elif var_name == 'met':
                    vals = {}
                    met_vars = ['tm_pres', 'tm_tpot', 'tm_tamb', 'tm_rain',
                                'tm_mixd', 'tm_relh', 'tm_sphu', 'tm_mixr',
                                'tm_dswf', 'tm_terr']
                    met_lets = ['P', 'O', 'A', 'R', 'D', 'H', 'E', 'W', 'S', 'T']
                    for j in range(10):
                        if val.find(met_lets[j]) == -1:
                            vals[met_vars[j]] = 0
                        else:
                            vals[met_vars[j]] = 1
                    non_d[var_name] = vals
                else:
                    non_d[var_name] = val
            if not non_d:
                # FIXME: Not sure why I put z in here - look into later
                z = None  # analysis:ignore
            return {'lat': lat, 'lon': lon, 'met': met, 'ver': ver, 'dur': dur,
                    'alt_type': alt_type, 'alt': alt, 'yr': yr, 'mo': mo,
                    'day': day, 'hr': hr, 'non_d': non_d}
        except (ValueError, IndexError):
            return False

    def check_run(self, run):
        """Checks if HYSPLIT 'run' is a valid name.
        Boolean return.
        """
        if self.get_var(run) is False:
            return False
        return True

    def check_ex(self, run, mkdir=False):
        """Checks if HYSPLIT output file and directory exists.
        If mkdir is True, will create the directory if needed.
        Returns None if bad run name.
        Returns False if valid name and file doesn't exist.
        Returns True if valid name and file does exist.
        """
        if not self.check_run(run):
            return None
        run_dir = self.run_dir(run)
        if not (os.path.isdir(
                os.path.join(self.HyS.out_dir, self.get_var(run)['met']))) \
                and mkdir:
            os.mkdir(os.path.join(self.HyS.out_dir, self.get_var(run)['met']))
        if not (os.path.isdir(run_dir)):
            os.mkdir(run_dir)
        if os.path.isfile(os.path.join(run_dir, run)):
            return True
        else:
            return False

    def _list_runs(self):
        """Original method for adding runs to a list by user input.
        Returns a list of run names, or False if no valid runs to return (need
        to rerun input).
        NOTE: This is a very old method from an earlier version intended to
        add runs for a specific situation. There are much better ways to do this
        now, but it is left in for the time being as a simple way to add runs
        in interactive mode.
        """
        # -Get needed runs-
        print 'Start HYSPLIT Runs for:'
        print ''
        print 'Number of Locations:'
        while True:
            try:
                num_loc = input()
                if isinstance(num_loc, int) and num_loc >= -1:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        if num_loc == -1:
            return False
        print ''
        print 'Location (Lat, Long to two decimal places as [X.XX,X.XX]):'
        print '  Note that south lat and west long should be negative'
        lat = []
        lon = []
        for i in range(num_loc):
            print 'Location %s:' % (i + 1)
            while True:
                try:
                    lat_lon = input()
                    if isinstance(lat_lon, list) and len(lat_lon) == 2:
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            lat.append(lat_lon[0])
            lon.append(lat_lon[1])
        print ''
        print '0: Gdas1'
        print '1: Gdas0p5'
        print '2: Edas40'
        print '3: Reanalysis'
        print '4: Coamps4'
        print 'Met Dataset:'
        while True:
            try:
                met_int = input()
                if isinstance(met_int, int) and 0 <= met_int < 5:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        met_dict = {0: 'gdas1', 1: 'gdas0p5', 2: 'edas40', 3: 'rean', 4: 'cmp4'}
        if met_int == 4:
            print 'WARNING: Naming of HYSPLIT COAMPS met datasets are not'
            print 'unique as the grid varies based on the COAMPS run.'
            print 'Care should be taken to ensure the met datasets for the'
            print 'proper run are included in the HYSPLIT met files directory.'
        met = met_dict[met_int]
        print ''
        print 'Vertical Motion option:'
        while True:
            try:
                ver = input()
                if isinstance(ver, int) and 0 <= ver <= 5:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Duration hours (negative for backtrajectories):'
        while True:
            try:
                dur = input()
                if isinstance(dur, int):
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Altitude Type, 0:AGL 1:MSL:'
        while True:
            try:
                alt_type = input()
                if isinstance(alt_type, int) and 0 <= alt_type <= 1:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Number of Altitudes:'
        while True:
            try:
                num_alt = input()
                if isinstance(num_alt, int):
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Altitude (meters):'
        alt = []
        for i in range(num_alt):
            print 'Altitude ' + str(i + 1) + ':'
            while True:
                try:
                    tmp = input()
                    if isinstance(tmp, int):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            alt.append(tmp)
        print ''
        print 'Date Range (Start Date, End Date as [YYYY,MM,DD,YYYY,MM,DD]):'
        while True:
            try:
                date_range = input()
                if isinstance(date_range, list) and len(date_range) == 6:
                    if self.check_date(str(date_range[0]) +
                                               str(date_range[1]) +
                                               str(date_range[2])) == True \
                            and self.check_date(str(date_range[3]) +
                                                        str(date_range[4]) +
                                                        str(date_range[5])) == True:
                        break
                    else:
                        print 'Invalid Date'
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        b_yr, b_mo, b_day, e_yr, e_mo, e_day = date_range
        print ''
        print 'Run at Hours each day: ([X,X,X] etc.) or'
        print '-1 for [0,3,6,9,12,15,18,21]:'
        while True:
            try:
                hrs = input()
                if hrs == -1:
                    hrs = [0, 3, 6, 9, 12, 15, 18, 21]
                    break
                else:
                    if isinstance(hrs, list) and len(hrs) <= 24:
                        break
                    else:
                        print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        print ''
        print 'Change any Default options? (0:No 1:Yes):'
        while True:
            try:
                nd = input()
                if type(nd).__name__ == 'int' and 0 <= nd <= 1:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        if nd:
            print 'Enter value for Non-Default run options.'
            print 'Enter "-1" to leave as default.'
            non_d = {}
            while True:
                try:
                    met_ = input('met (\'string\'): ')
                    if isinstance(met, str):
                        break
                    else:
                        if type(met).__name__ == 'int' and met == -1:
                            break
                        else:
                            print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if met_ != -1:
                non_d['met'] = met_
            while True:
                try:
                    mgmin = input('subgrid size (integer): ')
                    if isinstance(mgmin, int) and (mgmin >= 0 or mgmin == -1):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if mgmin != -1:
                non_d['mgmin'] = mgmin
            while True:
                try:
                    tratio = input('Grid cell ratio fraction (float): ')
                    if isinstance(tratio, float) and 0. <= tratio <= 1.:
                        break
                    else:
                        if isinstance(tratio, int) and tratio == -1:
                            break
                        else:
                            print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if tratio != -1:
                non_d['tratio'] = tratio
            if tratio == -1:
                while True:
                    try:
                        delt = input('integration time step (integer): ')
                        if isinstance(delt, int) \
                                and ((0 <= delt <= 60) or delt == -1):
                            break
                        else:
                            print 'Invalid Entry'
                    except (ValueError, SyntaxError, NameError):
                        print 'Invalid Entry'
                if delt != -1:
                    non_d['delt'] = delt
            while True:
                try:
                    tout = input(
                        'Endpoint output frequency in minutes (integer): ')
                    if isinstance(tout, int) and (tout >= 0 or tout == -1):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if tout != -1:
                non_d['tout'] = tout
            while True:
                try:
                    khmax = input(
                        'maximum trajectory duration in hours (integer): ')
                    if isinstance(khmax, int) and (khmax >= 0 or khmax == -1):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if khmax != -1:
                non_d['khmax'] = khmax
                non_d['mhrs'] = khmax
            while True:
                try:
                    mdom = \
                        input('top of model domain in meters AGL (integer): ')
                    if isinstance(mdom, int) and (mdom >= 0 or mdom == -1):
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            if mdom != -1:
                non_d['mdom'] = mdom
        else:
            non_d = None
        # -Print options and confirm-
        print ''
        print ''
        print 'Current HYSPLIT Run Request'
        print 'Location(s) (Lat(s), Lon(s)): ', lat, lon
        print 'Met Dataset: ', met
        print 'Vertical Motion Option: ', ver
        print 'Trajectory Duration (negative for backtrajectory): ', dur
        if alt_type == 0:
            tmp = ' AGL'
        else:
            tmp = ' MSL'
        print 'Altitude(s): ', alt, tmp
        print 'Date Range: ', date_range
        print 'Hours to Run at: ', hrs
        if nd:
            print 'Non-Default options: ', non_d
        print 'Is this correct? (0:No 1:Yes): '
        while True:
            try:
                correct = input()
                if isinstance(correct, int) and 0 <= correct <= 1:
                    break
                else:
                    print 'Invalid Entry'
            except (ValueError, SyntaxError, NameError):
                print 'Invalid Entry'
        if correct == 0:
            return False
        # -Generate each run time-
        # For n Runs (at one time each) of HYSPLIT there is an index which has
        # each time value at Yr[n], Mo[n] etc.
        n = 0  # The number of times HYSPLIT is to be run for each case
        yr = []
        mo = []
        day = []
        hr = []
        now = dt.date(b_yr, b_mo, b_day)
        end = dt.date(e_yr, e_mo, e_day)
        while now <= end:
            for i in hrs:
                yr.append(now.year)
                mo.append(now.month)
                day.append(now.day)
                hr.append(i)
                n += 1
            now = now + dt.timedelta(1)
        # -Save to list and return-
        ret = []
        for i_loc in range(num_loc):
            for i_alt in range(num_alt):
                for i in range(n):
                    ret.append(self.get_name(lat[i_loc], lon[i_loc], met,
                                             ver, dur, alt_type, alt[i_alt],
                                             yr[i], mo[i], day[i], hr[i],
                                             non_d))
        return ret

    def add_queue(self, ret_runset=False):
        """Queries user to add more runs to HYSPLIT queue.
        Parameters:
            ret_runset:     If True, will return a list of all newly added
                            run names.
                            If False, will return None
        """
        runset = []
        while True:
            # User add runs
            rslt = self._list_runs()
            more = 0
            if rslt:
                runset += rslt
            else:
                print 'Warning: No runs added'
            # Check if more runs are needed
            while True:
                try:
                    more = input('Add more runs? (0:No 1:Yes): ')
                    if isinstance(more, int) and 0 <= more <= 1:
                        break
                    else:
                        print 'Invalid Entry'
                except (ValueError, SyntaxError, NameError):
                    print 'Invalid Entry'
            # Break user add runs loop if done
            if more == 0:
                break
        # Open Queue.txt to append new runs
        q_file = open(os.path.join(self.HyS.var_dir, 'Queue.txt'), 'a')
        for r in runset:
            q_file.write(r + '\n')
        print '%s HYSPLIT runs added to Queue' % len(runset)
        q_file.close()
        # Return runset if requested
        if ret_runset:
            return runset
{'fraction_non_alphanumeric': 0.09565866865838857, 'fraction_numerical': 0.009709644290915882, 'mean_word_length': 1.1708958248885286, 'pattern_counts': {'":': 2, '<': 51, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3881559', 'n_tokens_mistral': 15187, 'n_tokens_neox': 14205, 'n_words': 4605}