<jupyter_start><jupyter_text>Histograms & Rolling Averages<jupyter_code>mean_temp = 0.5*(w['TMIN']+w['TMAX'])[mask]

import ipywidgets # interactivity

# make our data look less noisy with rolling averages
@ipywidgets.interact(window=(1,40,1))
def make_plot(window):
    mean_temp_avg = mean_temp.rolling(window=window).mean()
    plt.plot(mean_temp, marker='.', linewidth=0.5, alpha=0.5)
    plt.plot(mean_temp_avg, marker='.', linewidth=1.5)
    plt.xlabel('Date')
    plt.ylabel('Mean Daily Temp in F')
w.keys()
precp = w['PRCP'][mask]

# we want to format our dates correctly
import matplotlib.dates as mdates

set_ind = False
for k in w.keys():
    if k.find('DATE') != -1: # have we indexed by date yet?
        set_ind = True

if set_ind: w.set_index('DATE', inplace=True)
#w['PRCP']
#w


names = ['SteveBob', 'Jerry', 'Frank']
for n in names:
    print(n.find('Bob'))
# because we have re-indexed, lets redefine our arrays
mask = w['NAME'] == 'CHAMPAIGN 3 S, IL US'
mean_temp = 0.5*(w['TMIN']+w['TMAX'])[mask]
precp = w['PRCP'][mask]

@ipywidgets.interact(window=(1,60,2))
def make_plot(window):
    fig, ax = plt.subplots(1,2, figsize=(10,4))
    # (2) This was right-handed binning
    #mean_temp_avg = mean_temp.rolling(window=window).mean()
    mean_temp_avg = mean_temp.rolling(window=window, center=True).mean()
    mean_temp.plot(ax=ax[0]) # using pandas to plot
    mean_temp_avg.plot(ax=ax[0])
    
    # (1) We tried this, but its not highlighting what
    #     we want
    #precp_avg = precp.rolling(window=window).mean()
    # (2) This was right-handed binning
    #precp_avg = precp.rolling(window=window).sum()
    precp_avg = precp.rolling(window=window, center=True).sum()
    precp.plot(ax=ax[1], marker='.', linewidth=0.5, alpha=0.5)
    precp_avg.plot(ax=ax[1], marker='.', linewidth=1.5)
    
    ax[1].set_xlabel('Date')
    ax[1].set_ylabel('Daily rainfall in inches')
    ax[0].set_xlabel('Date')
    ax[0].set_ylabel('Mean Daily Temp in F')
precp.rolling?
# now lets look at a binning example for our rainfall data
#  this is a strict histogram/rebinning exercise, NOT smoothing
#  Note: rolling averages/sums as we've been using, is somewhere
# between smoothing & binning

@ipywidgets.interact(window=(1,60,1), day_bins=(1,100,5))
def make_plot(window,day_bins):
    fig, ax = plt.subplots(1,2,figsize=(10,4))
    
    mean_temp_avg = mean_temp.rolling(window=window,center=True).mean()
    mean_temp.plot(ax=ax[0])
    mean_temp_avg.plot(ax=ax[0])
    
    precp.plot(ax=ax[1], marker='.', linewidth=0.5, alpha=0.5)
    precp_resampled = precp.resample(str(day_bins)+'D').sum()
    #                               day_bins = 5 => '5D', resampling by months is 'M'
    precp_resampled.plot(ax=ax[1], marker='.')
    
    ax[1].set_xlabel('Date')
    ax[1].set_ylabel('Summed Rainfall over ' + str(day_bins) + 'days, in Inches')
    ax[0].set_xlabel('Date')
    ax[0].set_ylabel('Mean Daily Temp in F')<jupyter_output><empty_output><jupyter_text>Take aways* rolling averages => smoothing-lite, or fancy binning => like on our left temp plot* on the right rainfall plot => HISTOGRAM* so, in binning or histograming we are more "truthful" to the originial data, where in smoothing we can "double count" data points across bins  Quick look at windowingToy example first<jupyter_code># window of 10 bins, constant data in the window
npoints = 10
x = np.arange(0,npoints)
y = np.repeat(1,npoints)
plt.plot(x,y,'o', label='Original Data')

# so lets say we really want to highlight the center
#  bins
plt.plot(x,y*np.bartlett(npoints),'o', label='Bartlett')
# also another type of window
plt.plot(x,y*np.hamming(npoints),'o',label='Hamming')
plt.legend()
# plot available windows
windows_avail = [None,'boxcar','triang','blackman','hamming',
                 'bartlett','parzen', 'bohman', 
                 'blackmanharris','nuttall','barthann']

@ipywidgets.interact(window=(1,100,1), window_type=windows_avail)
def make_plot(window, window_type):
    plt.plot(mean_temp)
    mean_temp_avg = mean_temp.rolling(window=window,center=True,win_type=window_type).mean()
    plt.plot(mean_temp_avg)<jupyter_output><empty_output><jupyter_text>Similar binning & Smoothing in 2D<jupyter_code>ufos = pd.read_csv("/Users/jillnaiman1/Downloads/ufo-scrubbed-geocoded-time-standardized-00.csv",
                  names = ["date", "city", "state", "country",
                          "shape", "duration_seconds", "duration",
                          "comment", "report_date", "latitude", "longitude"],
                  parse_dates = ["date", "report_date"])
# if you get a memory error, don't panic!
ufos
# quick plot
plt.plot(ufos['longitude'], ufos['latitude'],'.')
# colormaps
import matplotlib.cm as cm
plt.colormaps()
plt.scatter(ufos['longitude'],ufos['latitude'],c=np.log10(ufos['duration_seconds']))
plt.scatter(ufos['longitude'][0:10], ufos['latitude'][0:10],c=ufos['duration_seconds'][0:10])
# our data is hard to see, lets try some rebinning in 2D
plt.hexbin(ufos["longitude"], ufos["latitude"], ufos["duration_seconds"], gridsize=32,bins='log')
# almost the exact same thing we did with histograms before in 1D for our rainfall data
# can also smooth 2D images
import PIL.Image as Image
im = Image.open('/Users/jillnaiman1/Downloads/stitch_reworked.png')

fig,ax = plt.subplots(figsize=(5,5))
ax.imshow(im)
import PIL.ImageFilter as ImageFilter

myFilter = ImageFilter.GaussianBlur(radius=1)
smoothed_image = im.filter(myFilter)
fig,ax = plt.subplots(figsize=(5,5))
ax.imshow(smoothed_image)
data_im = np.array(im)
np.unique(data_im)
data_sm = np.array(smoothed_image)
np.unique(data_sm)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15388701965645477, 'fraction_numerical': 0.018948114042854614, 'mean_word_length': 4.625498007968128, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '11348', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2141, 'n_tokens_neox': 2045, 'n_words': 445, 'path': 'week04/.ipynb_checkpoints/Untitled-checkpoint.ipynb', 'repo_name': 'UIUC-iSchool-DataViz/spring2019online', 'size': '5647'}