<jupyter_start><jupyter_text>weather underground<jupyter_code># Leb_wug_sauce_dict = {'Leb_Dec_2018':{'sauce':'https://www.wunderground.com/history/monthly/us/nh/west-lebanon/KLEB/date/2018-12','yr':'2018'},
#                       'Leb_Jan_2019':{'sauce':'https://www.wunderground.com/history/monthly/us/nh/west-lebanon/KLEB/date/2019-1', 'yr':'2019'},
#                       'Leb_Feb_2019':{'sauce':'https://www.wunderground.com/history/monthly/us/nh/west-lebanon/KLEB/date/2019-2', 'yr':'2019'},
#                      }


# this is a bit weird, there is no direct link to the Foliage View station in Lebanon NH, but if you type out the address below by hand, 
# you will get that station, which is the default station for daily weather in Lebanon, NH. 
Leb_wug_sauce_dict = {'Leb_Dec_2018':{'sauce':'https://www.wunderground.com/history/monthly/us/nh/west-lebanon/KNHLEBAN27/date/2018-12',
                                      'yr':'2018'},
                      'Leb_Jan_2019':{'sauce':'https://www.wunderground.com/history/monthly/us/nh/west-lebanon/KNHLEBAN27/date/2019-1',
                                      'yr':'2019'},
                      'Leb_Feb_2019':{'sauce':'https://www.wunderground.com/history/monthly/us/nh/west-lebanon/KNHLEBAN27/date/2019-2',
                                      'yr':'2019'},
                     }


# again, had to figure out station code, and then put into html address by hand. 
Wapp_wug_sauce_dict = {'Wapp_Dec_2018':{'sauce':'https://www.wunderground.com/history/monthly/us/ny/wappingers-falls/KNYWAPPI41/date/2018-12',
                                        'yr':'2018'},
                       'Wapp_Jan_2019':{'sauce':'https://www.wunderground.com/history/monthly/us/ny/wappingers-falls/KNYWAPPI41/date/2019-01',
                                        'yr':'2019'},
                       'Wapp_Feb_2019':{'sauce':'https://www.wunderground.com/history/monthly/us/ny/wappingers-falls/KNYWAPPI41/date/2019-02',
                                        'yr':'2019'},
                      }

Lyn_wug_sauce_dict = {'Lynd_Dec_2018':{'sauce':'https://www.wunderground.com/history/monthly/us/oh/cleveland/KOHCLEVE65/date/2018-12',
                                       'yr':'2018'},
                       'Lynd_Jan_2019':{'sauce':'https://www.wunderground.com/history/monthly/us/oh/cleveland/KOHCLEVE65/date/2019-01',
                                        'yr':'2019'},
                       'Lynd_Feb_2019':{'sauce':'https://www.wunderground.com/history/monthly/us/oh/cleveland/KOHCLEVE65/date/2019-02',
                                        'yr':'2019'},
                      }
def fix_wug_date(month, day, year):
    """Add leading zeros for month and day"""
    
    if len(month)==1:
        month = '0' + month
    if len(day)==1:
        day = '0' + day
    return month + '-' + day + '-' + year
    
def get_wug_historical_data(dict_in):
    """Scrapes and parses weather.com historical monthly data."""
    """Note:
        Will get an error sometimes if browser doesn't load (error below). Just rerun.
        NoSuchElementException: Message: no such element: Unable to locate element: {"method":"class name","selector":"observation-table"}
          (Session info: chrome=73.0.3683.103)
          (Driver info: chromedriver=2.42.591059 (a3d9684d10d61aa0c45f6723b327283be1ebaad8),platform=Mac OS X 10.14.4 x86_64)"""

    all_dfs = []
    
    for key in dict_in.keys():

        # open browser and wait for it to load
        with webdriver.Chrome() as browser:
            browser.get(dict_in[key]['sauce'])
            time.sleep(30)

            weather_table = browser.find_element_by_class_name("observation-table")
            mytable = pd.read_html(weather_table.get_attribute('outerHTML'))

            #get the month and format it as 2-digit string
            month_str = mytable[0].iloc[0,0].split(' ')[0]
            month_num = datetime.strptime(month_str,'%b').strftime('%m')
            
        # rework headers in individual dfs
        for i in range(1,8):
            # make first row into header 
            new_header = mytable[i].iloc[0]
            mytable[i] = mytable[i][1:]
            mytable[i].columns = new_header

            # add hierarchical header from first df
            top_header = mytable[0].columns[i-1]
            mytable[i] = pd.concat({top_header:mytable[i]}, axis = 1)

        #combine all dfs, one for each temp, dew_point, humidity etc. 
        final_table = mytable[1]
        for i in range(2,8):
            final_table = final_table.join(mytable[i]) # I do not fully understand why this works

        # make this standard
        final_table.rename(columns = {'Temperature (° F)': 'Temperature'})
        
        # create new date col, set as index, drop old date (Time) column
        date_srs = final_table[('Time', month_str)].apply(lambda d: fix_wug_date(month_num, d, dict_in[key]['yr'])  ) 
        date_srs = date_srs.rename('date')
        final_table.set_index(date_srs, inplace = True)      
        final_table.drop(columns = 'Time', inplace = True)# createa an error, but seems fine
        
        final_table.rename(columns={'Temperature (° F)':'Temperature'}, inplace = True)
        
        all_dfs.append(final_table)
    
        
    final_df = pd.concat(all_dfs).drop_duplicates()
            
    return final_df
wug_Leb_df = get_wug_historical_data(Leb_wug_sauce_dict)
wug_Leb_df.to_csv(path_or_buf = 'historical_weather_data/Leb_wug_weather_data.csv')
wug_Wapp_df = get_wug_historical_data(Wapp_wug_sauce_dict)
wug_Wapp_df.to_csv(path_or_buf = 'historical_weather_data/Wapp_wug_weather_data.csv')
wug_Lyn_df = get_wug_historical_data(Lyn_wug_sauce_dict)
wug_Lyn_df.to_csv(path_or_buf = 'historical_weather_data/Lyn_wug_weather_data.csv')<jupyter_output>/anaconda/lib/python3.6/site-packages/pandas/core/generic.py:3812: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.
  new_axis = axis.drop(labels, errors=errors)
/anaconda/lib/python3.6/site-packages/pandas/core/generic.py:3812: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.
  new_axis = axis.drop(labels, errors=errors)
/anaconda/lib/python3.6/site-packages/pandas/core/generic.py:3812: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.
  new_axis = axis.drop(labels, errors=errors)<jupyter_text>accuweather<jupyter_code>awc_Leb_sauces = {'Leb_Dec_2018_awc':{'sauce': 'https://www.accuweather.com/en/us/lebanon-nh/03766/month/1289_pc?monyr=12/01/2018', 
                                  'yr':'2018'},
                  'Leb_Jan_2019_awc':{'sauce': 'https://www.accuweather.com/en/us/lebanon-nh/03766/month/1289_pc?monyr=01/01/2019', 
                                  'yr':'2019'},
                  'Leb_Feb_2019_awc':{'sauce': 'https://www.accuweather.com/en/us/lebanon-nh/03766/month/1289_pc?monyr=02/01/2019', 
                                  'yr':'2019'},
                 }
              
awc_Wapp_sauces = {'Wapp_Dec_2018_awc': {'sauce': 'https://www.accuweather.com/en/us/wappingers-falls-ny/03766/month/4751_pc?monyr=12/01/2018', 
                                     'yr':'2018'},
                   'Wapp_Jan_2019_awc': {'sauce': 'https://www.accuweather.com/en/us/wappingers-falls-ny/03766/month/4751_pc?monyr=01/01/2019', 
                                     'yr':'2019'},
                   'Wapp_Feb_2019_awc': {'sauce': 'https://www.accuweather.com/en/us/wappingers-falls-ny/03766/month/4751_pc?monyr=02/01/2019', 
                                      'yr':'2019'},
                    }
               
awc_Lyn_sauces = {'Lynd_Dec_2018_awc': {'sauce': 'https://www.accuweather.com/en/us/cleveland-oh/44124/month/18910_pc?monyr=12/01/2018', 
                                    'yr':'2018'},
                  'Lynd_Jan_2019_awc': {'sauce': 'https://www.accuweather.com/en/us/cleveland-oh/44124/month/18910_pc?monyr=01/01/2019', 
                                    'yr':'2019'},
                  'Lynd_Feb_2019_awc': {'sauce': 'https://www.accuweather.com/en/us/cleveland-oh/44124/month/18910_pc?monyr=02/01/2019', 
                                    'yr':'2019'},
                 }
def parse_historical_awc(entry, year): 
    """Parses accuweather.com historical monthly data."""
    date, actual_and_ave_temps = entry.split('  Actual Temp  ')
    actual_temps, ave_temps = actual_and_ave_temps.split('  Hist. Avg.  ')
    
    date = datetime.strptime(date + ' ' + year,'%a %b %d %Y')
    date = date.strftime('%m-%d-%Y')
    
    actual_temps = actual_temps.split('  /')
    high_temp = actual_temps[0].rstrip('°')
    low_temp = actual_temps[1].rstrip('°')
    
    ave_temps = ave_temps.split('/')
    ave_high_temp = ave_temps[0].rstrip('°')
    ave_low_temp = ave_temps[1].rstrip('°')
    
    return {'date':date, 'Max': high_temp, 'Min': low_temp, 'Max_Ave': ave_high_temp, 'Min_Ave': ave_low_temp}
def scrape_historical_awc_data(dict_of_sauces):
    """Scrapes accuweather.com historical monthly data."""
    all_dfs = []
    
    for place_n_date in dict_of_sauces.keys(): 
        
        with webdriver.Chrome() as browser:

            browser.get(dict_of_sauces[place_n_date]['sauce'])

            weather_table = browser.find_element_by_id("content")
            acw_table = pd.read_html(weather_table.get_attribute('outerHTML'))

        # flatten table
        acw_table_flat_vals = acw_table[0].values.flatten()

        # make df and set index
        
        awc_final_parsed_table = []
        prior_date = 0
        recording = False
        
        for v in acw_table_flat_vals:
            date = v.split('  Actual Temp  ')[0]
         #   print(date)
            curr_day = date.split(' ')[2]
        #    print(date)  
        
            if recording:
                if curr_day =='1': # if recording and date = 1, this is a second new month -> break
                    break
                else:
                    entry = parse_historical_awc(v, year = dict_of_sauces[place_n_date]['yr'])     
                    awc_final_parsed_table.append(entry)
            else:
                if curr_day =='1': # if not recording and date = 1, this is the first of month -> start recording.
                    entry = parse_historical_awc(v, year = dict_of_sauces[place_n_date]['yr'])     
                    awc_final_parsed_table.append(entry)   
                    recording = True
 
        awc_df = pd.DataFrame(awc_final_parsed_table)   
      #  print(awc_df)
        awc_df = awc_df.set_index('date')

        
        
        #add hiearchical column header, and reorder columns
        awc_df = pd.concat({'Temperature':awc_df}, axis = 1)
        awc_df = awc_df[[('Temperature','Max'),('Temperature','Min'),('Temperature','Max_Ave'),('Temperature','Min_Ave')]]

        all_dfs.append(awc_df)
   
  #  print(all_dfs)
    final_df = pd.concat(all_dfs)#.drop_duplicates()

    
    return final_df
Leb_df = scrape_historical_awc_data(awc_Leb_sauces)
Leb_df.to_csv(path_or_buf='historical_weather_data/Leb_awc_historical_weather_data.csv')

Wapp_df = scrape_historical_awc_data(awc_Wapp_sauces)
Wapp_df.to_csv(path_or_buf='historical_weather_data/Wapp_awc_historical_weather_data.csv')

Lyn_df = scrape_historical_awc_data(awc_Lyn_sauces)
Lyn_df.to_csv(path_or_buf='historical_weather_data/Lyn_awc_historical_weather_data.csv')<jupyter_output><empty_output><jupyter_text>weather.com<jupyter_code>wdc_Leb_sauces = {'Leb_Dec_2018_wdc':{'sauce': 'https://weather.com/weather/monthly/l/USID0448:27:US', 
                                      'month':'Dec','yr':'2018'},
                  'Leb_Jan_2019_wdc':{'sauce': 'https://weather.com/weather/monthly/l/USID0448:27:US', 
                                      'month':'Jan','yr':'2019'},
                  'Leb_Feb_2019_awc':{'sauce': 'https://weather.com/weather/monthly/l/USID0448:27:US', 
                                      'month':'Feb','yr':'2019'},
                 }
              
wdc_Wapp_sauces = {'Wapp_Dec_2018_wdc': {'sauce': 'https://weather.com/weather/monthly/l/USNY1512:1:US', 
                                         'month':'Dec','yr':'2018'},
                   'Wapp_Jan_2019_wdc': {'sauce': 'https://weather.com/weather/monthly/l/USNY1512:1:US', 
                                          'month':'Jan','yr':'2019'},
                   'Wapp_Feb_2019_wdc': {'sauce': 'https://weather.com/weather/monthly/l/USNY1512:1:US', 
                                          'month':'Feb','yr':'2019'},
                    }
               
wdc_Lyn_sauces = {'Lynd_Dec_2018_wdc': {'sauce': 'https://weather.com/weather/monthly/l/44124:4:US', 
                                       'month':'Dec','yr':'2018'},
                  'Lynd_Jan_2019_wdc': {'sauce': 'https://weather.com/weather/monthly/l/44124:4:US', 
                                       'month':'Jan','yr':'2019'},
                  'Lynd_Feb_2019_wdc': {'sauce': 'https://weather.com/weather/monthly/l/44124:4:US', 
                                        'month':'Feb','yr':'2019'},
                 }
def get_wdc_historical_data(sauce, month, year):
    """Scrapes weatherunderground.com historical monthly data."""
    month_and_year = month + ' ' + year

    with webdriver.Chrome() as browser:
        browser.get(sauce)

        time.sleep(10) # wait for the page to load. 

        month_button = Select(browser.find_element_by_id("month-picker"))

        i = 0
        while True:

            month_button.select_by_visible_text(month_and_year)

            time.sleep(10) # wait for the new page to load

            test_but = month_button.first_selected_option.text

            if month_button.first_selected_option.text == month_and_year:
                break

            i+=1
            
            if i==10:
                print('Tried to get weather.com historical data 10 times, breaking.')

        wdc_weather_table = browser.find_element_by_class_name("forecast-monthly")
        wdc_table = wdc_weather_table.get_attribute('outerHTML')

        soup = BeautifulSoup(wdc_table,"lxml")

    return soup.findAll("div", {"class":"dayCell opaque"}) # "opaque" days are the past days
def parse_wdc_historical_monthly_date(data_in, month, year):
    """Parses weatherunderground.com historical monthly data."""
    final_list = []
    prior_date = 0

    recording = False

    for d in data_in:

        curr_day = d.find("div",{"class":"date"}).text
        # note to self: there is not much other info in this page other than low and hi temp
        # they seems to have the nightly cloud cover as well, but that is it. 
        curr_hi_temp = d.find("div",{"class":"hi"}).text  \
                        .rstrip('°')
        curr_low_temp = d.find("div",{"class":"low"}).text \
                         .rstrip('°')
  
        if recording:
            if curr_day =='1': # if recording and date = 1, this is a second new month -> break
                break
            else:
                curr_date = month + '-' + curr_day + '-' + year
                curr_date = datetime.strptime(curr_date,'%b-%d-%Y')
                curr_date = curr_date.strftime('%m-%d-%Y')
                final_list.append({'date':curr_date, 'Max':curr_hi_temp, 'Min':curr_low_temp})
        else:
            if curr_day =='1': # if not recording and date = 1, this is the first of month -> start recording.
                recording = True
                curr_date = month + '-' + curr_day + '-' + year
                curr_date = datetime.strptime(curr_date,'%b-%d-%Y')
                curr_date = curr_date.strftime('%m-%d-%Y')
                final_list.append({'date':curr_date, 'Max':curr_hi_temp, 'Min':curr_low_temp})        
               
    final_df = pd.DataFrame(final_list)    
    final_df.set_index(keys = 'date', inplace = True)
    final_df = pd.concat({'Temperature':final_df}, axis = 1)
    
    return final_df
final_dfs = []

for k in wdc_Leb_sauces:
    wdc_scraped_data = get_wdc_historical_data(sauce = wdc_Leb_sauces[k]['sauce'], 
                                               month = wdc_Leb_sauces[k]['month'], 
                                               year = wdc_Leb_sauces[k]['yr'])
    wdc_df = parse_wdc_historical_monthly_date(data_in = wdc_scraped_data, month = wdc_Leb_sauces[k]['month'], year = wdc_Leb_sauces[k]['yr'])
    final_dfs.append(wdc_df)
    
Leb_wdc_historical_df = pd.concat(final_dfs)
Leb_wdc_historical_df.to_csv(path_or_buf = 'historical_weather_data/Leb_wdc_weather_data.csv')
final_dfs = []

for k in wdc_Wapp_sauces:
    wdc_scraped_data = get_wdc_historical_data(sauce = wdc_Wapp_sauces[k]['sauce'], 
                                               month = wdc_Wapp_sauces[k]['month'], 
                                               year = wdc_Wapp_sauces[k]['yr'])
    wdc_df = parse_wdc_historical_monthly_date(data_in = wdc_scraped_data, month = wdc_Wapp_sauces[k]['month'], year = wdc_Wapp_sauces[k]['yr'])
    final_dfs.append(wdc_df)
    
Wapp_wdc_historical_df = pd.concat(final_dfs)
Wapp_wdc_historical_df.to_csv(path_or_buf = 'historical_weather_data/Wapp_wdc_weather_data.csv')
final_dfs = []

for k in wdc_Lyn_sauces:
    wdc_scraped_data = get_wdc_historical_data(sauce = wdc_Lyn_sauces[k]['sauce'], 
                                               month = wdc_Lyn_sauces[k]['month'], 
                                               year = wdc_Lyn_sauces[k]['yr'])
    wdc_df = parse_wdc_historical_monthly_date(data_in = wdc_scraped_data, month = wdc_Lyn_sauces[k]['month'], year = wdc_Lyn_sauces[k]['yr'])
    final_dfs.append(wdc_df)
    
Lyn_wdc_historical_df = pd.concat(final_dfs)
Lyn_wdc_historical_df.to_csv(path_or_buf = 'historical_weather_data/Lyn_wdc_weather_data.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13248699737151165, 'fraction_numerical': 0.03651921033499245, 'mean_word_length': 2.2998708248754385, 'pattern_counts': {'":': 6, '<': 12, '<?xml version=': 0, '>': 16, 'https://': 30, 'lorem ipsum': 0, 'www.': 21, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '119271', 'license': 'MIT', 'n_tokens_mistral': 6761, 'n_tokens_neox': 6030, 'n_words': 892, 'path': 'Comparing_long-term_weather_forecasts/get_historical_weather_data.ipynb', 'repo_name': 'TheDohn/TheDohn.github.io', 'size': '17881'}