<jupyter_start><jupyter_text>Sensor data!<jupyter_code>def pad(x):
    time = x.astype(str).zfill(4)
    return time[:2] + ':' + time[2:]
#contains headers for all data (I edited "nothing" column in header file b/c it's just 0's)

filefolder = 'data/sensor_data/colorado6months/'
desired_file = '20140401.csv'

def return_sensor_data(desired_file, filefolder):
    '''Input: desired file and folder
    Output: sensor data pandas dataframe
    Usage: filefolder = 'data/sensor_data/colorado6months/'
    desired_file = '20140401.csv'
    return_sensor_data(myfile, filefolder)'''
    
    df_header = pd.read_csv(filefolder + 'header.csv')
    headers = df_header.columns
    df_sensor = pd.read_csv(filefolder + desired_file,header=None) #sensors
    df_sensor.columns = headers
    df_sensor['MST'] = df_sensor['MST'].map(pad)
    #make a datetime index:
    df_sensor['datetime'] = pd.to_datetime(df_sensor['Year'].astype(str)+
                                           df_sensor['DOY'].astype(str)+
                                           df_sensor['MST'].astype(str), 
                                           format='%Y%j%H:%M')
    #convert to UTC time, website shows that they disregard daylight savings time
    df_sensor['datetime'] = df_sensor['datetime'] + pd.Timedelta(hours=7) 
    df_sensor.set_index(['datetime'],inplace=True) #set created column as index ...
    df_sensor = df_sensor.resample('H')# ... so that we can resample it (hourly)
    return df_sensor  

return_sensor_data(desired_file, filefolder)<jupyter_output><empty_output><jupyter_text>PV Output Data!<jupyter_code>filefolder = 'data/pvoutput/pvoutput6months/'
desired_file = '20140401.csv'

def return_pvoutput_data(desired_file, filefolder):
    '''Input: Desired file and folder
    Output: PV Output dataframe
    Usage: filefolder = 'data/pvoutput/pvoutput6months/'
    desired_file = '20140401.csv'
    return_pvoutput_data(desired_file, filefolder)'''
    df_output = pd.read_csv(filefolder + desired_file) #pvoutput
    #df_output['datetime'] = df_output['datetime'] + pd.Timedelta(hours=6) #convert to utc time
    df_output['datetime'] = df_output['datetime'].apply(pd.to_datetime)
    df_output['datetime'] = df_output['datetime'] + pd.Timedelta(hours=6) #convert to utc time
    df_output.set_index(['datetime'],inplace=True) #set created column as index ...
    df_output = df_output.resample('H') # ...so that we can resample it (hourly)
    return df_output[['Power']]

return_pvoutput_data(desired_file, filefolder)<jupyter_output><empty_output><jupyter_text>Moooor Satellite Data<jupyter_code>%run data_helper_functions.py
from data_helper_functions import plot_satellite_image, return_satellite_data
filename  = 'data/satellite/colorado/summer6months/data/goes15.2014.091.110018.BAND_06.nc'
plot_satellite_image(filename)
lons, lats, data = return_satellite_data(filename)

plt.figure(figsize=(8, 8))
imgplot = plt.imshow(data)
imgplot.set_interpolation('none')<jupyter_output><empty_output><jupyter_text>Find files based on channel and datetime for satellite data<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline
from datetime import datetime
from data_helper_functions import plot_satellite_image, \
    return_satellite_data, find_closest_date, find_file_details, \
    find_closest_date, find_desired_file
filefolder = "data/satellite/colorado/summer6months/data/"
desired_datetime = datetime(2014, 5, 5, 22) #year, month, day [, hour, minute, second]
desired_channel = 'BAND_01' #channel

sat_image_file = filefolder+find_desired_file(desired_datetime, desired_channel, filefolder)

lons, lats, data = return_satellite_data(sat_image_file)
plt.figure(figsize=(8, 8))
imgplot = plt.imshow(data)
imgplot.set_interpolation('none')
plt.show()<jupyter_output><empty_output><jupyter_text>Get correct file for Sensor Data and PV Output!Now we have to do something similar to what we did for satellite data<jupyter_code>import os
from datetime import datetime

def find_file_from_date(desired_datetime, filefolder, filetype = 'csv'):
    '''Input: datetime, folder, filetype; Output: file'''
    data_files = os.listdir(filefolder)

    list_of_files = [] #contains all the files
    list_of_files_details = [] #contains information collected from filename
    for myfile in data_files:
        if (myfile[-2:] == filetype or myfile[-3:] == filetype): #only get netCDF by default
            list_of_files.append(myfile)
            list_of_files_details.append(myfile.split('.'))

    for i,val in enumerate(list_of_files_details):
        try:   
            if datetime.strptime(list_of_files_details[i][0], '%Y%m%d') == desired_datetime:
                return list_of_files[i]
        except:
            pass
        
filefolder = "data/pvoutput/pvoutput6months/"
desired_datetime = datetime(2014, 5, 5) #year, month, day [, hour, minute, second]

find_file_from_date(desired_datetime, filefolder)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11268174474959612, 'fraction_numerical': 0.017366720516962843, 'mean_word_length': 4.552690582959642, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '598428', 'license': 'Apache-2.0', 'n_tokens_mistral': 1696, 'n_tokens_neox': 1598, 'n_words': 338, 'path': 'IPnotebooks/less-important-IPNBs/building-data-functions.ipynb', 'repo_name': 'scottlittle/solar-sensors', 'size': '4952'}