<commit_before>#!/bin/env python

# coding: utf-8

# Severe Weather Forecasting with Python and Data Science Tools: Interactive Demo
# <NAME>, University of Oklahoma and NCAR
# Introduction
# Severe weather forecasting has entered an age of unprecedented access to large model and observational datasets with even greater hordes of data in the pipeline. With multiple ensembles of convection-allowing models available and an increasing variety of observations derived from radar, satellite, surface, upper air, and crowd-sourcing, forecasters can easily be overwhelmed with guidance. Without ways to organize, synthesize, and visualize the data in a useful manner for forecasters, the pile of new models and observations will languish unused and will not fulfill their full potential. An even worse outcome would be to take the human forecasters completely out of the loop and trust the models, which is a way fraught with peril. Data science tools offer ways to synthesize essential information from many disparate data sources while also quantifying uncertainty. When forecasters use the tools properly, they can identify potential hazards and the associated spatial and time uncertainties more quickly by using the output of the tools to help target their domain knowledge.
# This module demonstrates how data science tools from the image processing and machine learning families can be used to create a forecast of severe hail. It aims to teach the advantages, challenges, and limitations of these tools through hands-on interaction.
#  

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Polygon, PathPatch
from matplotlib.collections import PatchCollection
from datetime import datetime, timedelta
from mpl_toolkits.basemap import Basemap
from scipy.ndimage import gaussian_filter, find_objects
from copy import deepcopy
from mysavfig import mysavfig
import pdb, sys, argparse, os


# In[2]:

from hagelslag.processing import EnhancedWatershed
from hagelslag.data import ModelOutput
from hagelslag.processing import ObjectMatcher, shifted_centroid_distance, centroid_distance, closest_distance
from hagelslag.processing import STObject

parser = argparse.ArgumentParser(description='object tracker')
parser.add_argument('-m', '--member', type=str, help='member description (e.g. 1km_pbl7)', default='1km_on_3km_pbl1')
parser.add_argument('-d', '--date', help='date yyyymmddhh', default='2005011312')
parser.add_argument('-f', '--field', default='UP_HELI_MAX03', help='field in which to find objects')
parser.add_argument('-t','--timethresh', type=int, default=3, help='time threshold (hours)')
parser.add_argument('-v','--verbose', action="store_true", help='print more output. useful for debugging')
args = parser.parse_args()

if args.verbose: print args

odir = '/glade/p/work/ahijevyc/hagelslag/out/'
model_path = "/glade/scratch/ahijevyc/VSE/"
ensemble_name = "NCAR"
run_date = datetime.strptime(args.date,'%Y%m%d%H')
member = args.member
field = args.field
start_date = run_date + timedelta(hours=10) # remember first time is usually all zeros

#    Attributes:
#        min_thresh (int): minimum pixel value for pixel to be part of a region
#        data_increment (int): quantization interval. Use 1 if you don't want to quantize
#        max_thresh (int): values greater than maxThresh are treated as the maximum threshold
#        size_threshold_pixels (int): clusters smaller than this threshold are ignored.
#        delta (int): maximum number of data increments the cluster is allowed to range over. Larger d results in clusters over larger scales.

# From ahij's config file.
if field == "MAX_UPDRAFT_HELICITY" or field == "UP_HELI_MAX03":
        params = {"min_thresh":25, "step":5, "max_thresh":250, "max_size":150, "delta":75, "min_size":0, "filter_size":1}
if field == "HAIL2D":
        params = {"min_thresh":0.025, "step":0.005, "max_thresh":0.1, "max_size":150, "delta":75, "min_size":0, "filter_size":1}
levels = params['min_thresh'] * np.arange(1,8)
levels = np.append(levels, params['min_thresh'] * 15)
model_watershed_params = (params['min_thresh'],params['step'],params['max_thresh'],params["max_size"],params["delta"])
<commit_msg>use strptime. better path selection. thinner map lines.<commit_after>#!/bin/env python

# coding: utf-8

# Severe Weather Forecasting with Python and Data Science Tools: Interactive Demo
# <NAME>, University of Oklahoma and NCAR
# Introduction
# Severe weather forecasting has entered an age of unprecedented access to large model and observational datasets with even greater hordes of data in the pipeline. With multiple ensembles of convection-allowing models available and an increasing variety of observations derived from radar, satellite, surface, upper air, and crowd-sourcing, forecasters can easily be overwhelmed with guidance. Without ways to organize, synthesize, and visualize the data in a useful manner for forecasters, the pile of new models and observations will languish unused and will not fulfill their full potential. An even worse outcome would be to take the human forecasters completely out of the loop and trust the models, which is a way fraught with peril. Data science tools offer ways to synthesize essential information from many disparate data sources while also quantifying uncertainty. When forecasters use the tools properly, they can identify potential hazards and the associated spatial and time uncertainties more quickly by using the output of the tools to help target their domain knowledge.
# This module demonstrates how data science tools from the image processing and machine learning families can be used to create a forecast of severe hail. It aims to teach the advantages, challenges, and limitations of these tools through hands-on interaction.
#  

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Polygon, PathPatch
from matplotlib.collections import PatchCollection
from datetime import datetime, timedelta
from mpl_toolkits.basemap import Basemap
from scipy.ndimage import gaussian_filter, find_objects
from copy import deepcopy
from mysavfig import mysavfig
import pdb, sys, argparse, os


# In[2]:

from hagelslag.processing import EnhancedWatershed
from hagelslag.data import ModelOutput
from hagelslag.processing import ObjectMatcher, shifted_centroid_distance, centroid_distance, closest_distance
from hagelslag.processing import STObject

parser = argparse.ArgumentParser(description='object tracker')
parser.add_argument('-m', '--member', type=str, help='member description (e.g. 1km_pbl7)', default='1km_on_3km_pbl1')
parser.add_argument('-d', '--date', help='date yyyymmddhh', default='2005011312')
parser.add_argument('-f', '--field', default='UP_HELI_MAX03', help='field in which to find objects')
parser.add_argument('-t','--timethresh', type=int, default=3, help='time threshold (hours)')
parser.add_argument('-v','--verbose', action="store_true", help='print more output. useful for debugging')
args = parser.parse_args()

if args.verbose: print args

odir = '/glade/p/work/ahijevyc/hagelslag/out/'
model_path = "/glade/scratch/ahijevyc/VSE/"
ensemble_name = "NCAR"
run_date = datetime.strptime(args.date,'%Y%m%d%H')
member = args.member
field = args.field
start_date = run_date + timedelta(hours=10) # remember first time is usually all zeros

#    Attributes:
#        min_thresh (int): minimum pixel value for pixel to be part of a region
#        data_increment (int): quantization interval. Use 1 if you don't want to quantize
#        max_thresh (int): values greater than maxThresh are treated as the maximum threshold
#        size_threshold_pixels (int): clusters smaller than this threshold are ignored.
#        delta (int): maximum number of data increments the cluster is allowed to range over. Larger d results in clusters over larger scales.

# From ahij's config file.
if field == "MAX_UPDRAFT_HELICITY" or field == "UP_HELI_MAX03":
        params = {"min_thresh":75, "step":5, "max_thresh":250, "max_size":50, "delta":75, "min_size":1, "filter_size":0}
if field == "HAIL2D":
        params = {"min_thresh":0.025, "step":0.005, "max_thresh":0.1, "max_size":150, "delta":75, "min_size":0, "filter_size":1}
levels = params['min_thresh'] * np.arange(1,8)
levels = np.append(levels, params['min_thresh'] * 15)
model_watershed_params = (params['min_thresh'],params['step'],params['max_thresh'],params["max_size"],params["delta"])

{'fraction_non_alphanumeric': 0.09019793765556477, 'fraction_numerical': 0.014104539528268342, 'mean_word_length': 5.5158301158301155, 'pattern_counts': {'":': 32, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1639999', 'n_tokens_mistral': 2486, 'n_tokens_neox': 2329, 'n_words': 935}