<jupyter_start><jupyter_text>Plot 30-year moving average $\Delta T$  for 1pctCO2 runs=======<jupyter_code>ns = np.array([[f, np.load('data/'+f, allow_pickle = True)] for f in os.listdir('data') if "1pctCO2" in f and '_ts.npy' in f])
below_ts = np.array([n for n in ns if np.mean(n[1][0]) < 200])
dates = [pd.to_datetime(n[1], errors='coerce') for n in ns[:,1]][0]
plt.figure("1pctCO2 runs")
for f, n in ns:
    if not isinstance(n[1,0], type(int)) and f not in below_ts[:,0]:
        w = 30
        plt.plot_date(dates[int(w/2)-1:int(len(n[0])-w/2)],
                      moving_average(n[0], w),
                      xdate=True)<jupyter_output><empty_output><jupyter_text>Plot 30-year moving average $\Delta T$  for abrupt-4x runs=======<jupyter_code>ns = np.array([[f, np.load('data/'+f, allow_pickle = True)] for f in os.listdir('data') if "abrupt" in f and '_ts.npy' in f])
below_ts = np.array([n[0] for n in ns if np.mean(n[1][0]) < 200])
short_ts = np.array([n[0] for n in ns if len(n[1][1]) < 1700])
dates = [d for d in [pd.to_datetime(n[1], errors='coerce') for n in ns[:,1]] if d[0] != 'NaT']
plt.figure("1pctCO2 runs")
for f, n in ns:
    if f not in below_ts and f not in short_ts:
        w = 30
        d = next(d for d in dates if len(d) == len(n[0]))
        plt.plot_date(d[int(w/2)-1:int(len(n[0])-w/2)],
                      moving_average(n[0], w),
                      xdate=True)
np.shape(dates[0])
def percentiles(ds, df):
    Area = get_area(ds, df)
    Var = ds.get(ds.variable_id)
    
    percentiles = []
    
    for t in range(len(Var.time)):
        area = Area[t]
        var = Var[t]
        time = Var.time[t]
        
        weights = np.reshape(area.data, (np.prod(np.shape(var)),1))
        vals = np.reshape(var.data, (np.prod(np.shape(var)),1))
        weights = np.array(weights[~np.isnan(vals)])
        vals = np.array(vals[~np.isnan(vals)])

        idx = np.argsort(vals)
        vals = np.take_along_axis(vals, idx, axis=0)
        sorted_weights = np.array(np.take_along_axis(weights, idx, axis=0))
        total = np.nansum(weights)

        i = 0
        low = 0
        N = len(weights)
        pcts = np.array([0.05, 0.17, 0.5, 0.83, 0.95])
        pct_vals = []
        for j in range(0,len(vals)):
            low = low + sorted_weights[j]
            high = low + sorted_weights[min(j+1,N)]
            # If the cumulative weights are nearest the next percentile
            # Then mark down the value
            if low/total < pcts[i] and high/total >= pcts[i]:
                pct_vals += [[pcts[i], vals[j]]]
                if i == np.shape(pcts)[0] - 1:
                    break
                i = i + 1
                
        percentiles += [time, pct_vals]
    
    return percentiles

def moving_average(x, w):
    return np.convolve(x, np.ones(w), 'valid') / w    
get_dims(ds)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16886726893676163, 'fraction_numerical': 0.028144544822793608, 'mean_word_length': 2.3283236994219654, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '288550', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1152, 'n_tokens_neox': 1091, 'n_words': 261, 'path': 'notebooks/pangeo.ipynb', 'repo_name': 'ghall3/pangeo_tests', 'size': '2878'}