<jupyter_start><jupyter_text>EDA 2 & 32. Now write a function which returns N entries of cities which have the highest variability in `AverageTemperature` in a specified time range.3. Plot the temperature development (and uncertainty) over time (aggregate as you see fit) of the top 4 entries of your function for a given time period and save the plot as a `.png` file.<jupyter_code>import pandas as pd
cols_subset = ['dt', 
               'AverageTemperature', 
               'AverageTemperatureUncertainty', 
               'City', 
               'Country']
pdf = pd.read_csv('./data/GlobalLandTemperaturesByCity.csv', \
                  parse_dates=['dt'], infer_datetime_format=True, 
                 usecols=cols_subset) \
                .dropna()
pdf.head()
country_key = 'Country' # 'Country_nr'
city_key = 'City' # 'City_nr'<jupyter_output><empty_output><jupyter_text>Date processing<jupyter_code>from datetime import datetime, timedelta

def get_datetime_range(date_str1, date_str2):

    date1 = list(map(int, date_str1.split('-'))) 
    date2 = list(map(int, date_str2.split('-')))     
    
    return [datetime(*date1),datetime(*date2)]

def get_nr_days(date_range):
    
    datediff = dt_range[1]-dt_range[0]
    
    return datediff.days+1

def _get_valid_range(date_range, threshold=0.75):
    
    tot_nr_days = get_nr_days(date_range)
    
    def meth(s):
        
        nr_days = s.size
        
        return nr_days/tot_nr_days >= threshold
    
    return meth<jupyter_output><empty_output><jupyter_text>Variability processing<jupyter_code># Quantile and IQR
qtile = lambda x : np.quantile(x, q=[0.25,0.75])
iqr = lambda x: np.abs(np.diff(qtile(x))[0])

# wrapper function in case want to use some other metric
def _variability_score(fnc):
    
    def meth(s):
    
        return fnc(s.values)
    
    return meth

variability_score = _variability_score(iqr)
# Filter function for the dates

def filter_by_dates(pdf, date_range):
    return pdf[(pdf['dt'] >= date_range[0]) & (pdf['dt'] <= date_range[1])]
def _get_highest_variability(fraction_of_obs=0.75):

    temper_key =  'AverageTemperature'
    
    grouping = [city_key, country_key]
    
    def meth(pdf, N,date_range):
    
        # Filter to the date range
        pdf_ = filter_by_dates(pdf, date_range)
        
        # Check if date range is contains valid number of events
        get_valid_range = _get_valid_range(date_range, fraction_of_obs)

        # Goupy on country and city, compute variability by IQR
        pdf_ =  pdf_.groupby(grouping)\
            .agg({temper_key:variability_score,'dt':get_valid_range})\
            .reset_index()\
            .rename(columns={temper_key:'variability'})
        
        # Get the largest N 
        pdf_ = pdf_[pdf_['dt']].nlargest(N,'variability')\
                    .drop(columns=['dt']) \
                    .reset_index(drop=True)
                
        return pdf_

    return meth

# Declare wrapper with threshold fraction for how full the time range should be
get_highest_variability = _get_highest_variability(0.0)<jupyter_output><empty_output><jupyter_text>Testing Define an arbitrary date range<jupyter_code>dt_range = get_datetime_range('1903-11-01','2013-08-01')<jupyter_output><empty_output><jupyter_text>Get the variability<jupyter_code>pdf1 = get_highest_variability(pdf, 10, dt_range)
pdf1.head()
pdf1.head()<jupyter_output><empty_output><jupyter_text>EDA 3Get the top 4 cities, and plot<jupyter_code>pdfN = get_highest_variability(pdf, 4, dt_range)
pdfN.head()
cities_list = pdfN.City.tolist()
cities_list.remove('Yakeshi')
print(cities_list)
# Get time series for these cities
pdf4 = pdf[pdf[city_key].isin(cities_list) & 
           pdf[country_key].isin(pdfN[country_key].tolist())].copy()
pdf4 = filter_by_dates(pdf4, dt_range)
pdf4.sort_values(by=[country_key, city_key, 'dt'], inplace=True)
# For the legend labels
top4 = pdf4[[country_key,city_key]].drop_duplicates().values<jupyter_output><empty_output><jupyter_text>Plotting<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
def get_label(df):
    return ", ".join(map(str,df[[city_key,country_key]].drop_duplicates().values[0]))


temper_key =  'AverageTemperature'

def plot_temp(df,timecol, temper_col, markerline, ax):
    
    ax.plot(df[timecol].values, 
            df[temper_col].values, 
            markerline, label=get_label(df))
    return ax
mask1 = (pdf4[country_key] == top4[0,0]) & (pdf4[city_key] == top4[0,1])
mask2 = (pdf4[country_key] == top4[1,0]) & (pdf4[city_key] == top4[1,1])
mask3 = (pdf4[country_key] == top4[2,0]) & (pdf4[city_key] == top4[2,1])
mask4 = (pdf4[country_key] == top4[3,0]) & (pdf4[city_key] == top4[3,1])
pdf4['decade'] = pdf4.dt.apply(lambda x: x.year-x.year%10)
pdf4g = pdf4.groupby([country_key, city_key, 'decade']).agg({temper_key:'mean'})\
    .reset_index() \
    .sort_values(by=[country_key, city_key, 'decade'])
mask1 = (pdf4g[country_key] == top4[0,0]) & (pdf4g[city_key] == top4[0,1])
mask2 = (pdf4g[country_key] == top4[1,0]) & (pdf4g[city_key] == top4[1,1])
mask3 = (pdf4g[country_key] == top4[2,0]) & (pdf4g[city_key] == top4[2,1])
mask4 = (pdf4g[country_key] == top4[3,0]) & (pdf4g[city_key] == top4[3,1])
fig, ax = plt.subplots(1,1,figsize=[20,6])

plot_temp(pdf4g[mask1], 'decade', temper_key, 'x-' , ax)
plot_temp(pdf4g[mask2], 'decade', temper_key, 'P--', ax)
plot_temp(pdf4g[mask3], 'decade', temper_key, 'X:' , ax)
plot_temp(pdf4g[mask4], 'decade', temper_key, 'o-.', ax)

plt.xlabel('date [year]')
plt.ylabel('temperature [deg. C]')

fig.suptitle(f'Top 4 cities by temperature variability, from {dt_range[0].year} - {dt_range[1].year}', fontsize=16)

plt.legend();

plt.grid();
fig.savefig('top4_cities_by_temperature_variability.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14110323361577037, 'fraction_numerical': 0.027840221338405673, 'mean_word_length': 4.091549295774648, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '586522', 'license': 'MIT', 'n_tokens_mistral': 2190, 'n_tokens_neox': 2108, 'n_words': 400, 'path': 'EDA_2_and_3.ipynb', 'repo_name': 'm-salewski/planetly_takehome', 'size': '5783'}