<jupyter_start><jupyter_text>EGCTF Main Module*This is the main-module which contains the EGCTF algorithm and its sub-functions. This module is to be executed prior to invoking the EGCTF algorithm. See the `test_one_case` script for an example of how to utilize this module. This file need not be modified unless the user wishes to make some deeper changes in the EGCTF algorithm.***External Dependencies**: Python modules: `scipy`, `sklearn`, `geopy` (requires `pip install`)<jupyter_code>%matplotlib inline
import pandas as pd
import matplotlib.pyplot  as plt
import matplotlib
import numpy as np
import copy
from datetime import datetime  
from datetime import timedelta  
import os
from scipy import interpolate
from scipy import optimize
import random
import geopy.distance
import numpy as np
from scipy.interpolate import Akima1DInterpolator
from sklearn import datasets, linear_model
import pickle
def mean_ensemble(ensTracks, ens_wts):
    """ Calculate mean-ensemble track (locations) given the enseble tracks and ensemble weights."""
    ens_wts = np.array(ens_wts) / np.sum(ens_wts)
    mens = np.zeros(ensTracks[0].shape)
    for tn in range(0,nens):
        mens = mens + ensTracks[tn]*ens_wts[tn]
    return mens

def track_position_akima(t_x, t_y, t_t, t):
    """ Return Akima interpolated track (ensemble, or best tracks) position at arbitrary time 't' given the track sparse data points (6 hrs interval) using Akima interpolation """
    akx = Akima1DInterpolator(t_t, t_x)
    aky = Akima1DInterpolator(t_t, t_y)
    return [akx(t), aky(t), t]


def recalc_ens_weights(known_storm_position, dud_obs, ens_wts, ensTracks, forecast_periods, senFPr, r_f):    
    """ (Re)Calculate set of optimal ensemble weights from (1) Known Storm positions, (2) Position of failed Observations,
        and (3) Previous set of ensemble weights
        Input Parameters (4) forcast periods, (5) sensor footprint radius are constant.
    """
    x0 = ens_wts # current ensemble weights used as intiial value in the optimization parameters
    lbd = np.ones((nens,1))*0.005/nens # lower bound for the optimization parameters
    ubd = np.ones((nens,1)) # upper bound
    bnds = np.hstack((lbd, ubd)) # ensemble weights must be positive
   
    # run the L-BFGS-B optimizer to find the optimal optimization parameter (ensemble weights) 
    optres = optimize.minimize(optfunc, x0, args=(ensTracks, forecast_periods, known_storm_position), method='L-BFGS-B', bounds = bnds, options={'maxiter': 10000})
    ens_wts = optres.x
    ens_wts = np.array(ens_wts) / np.sum(ens_wts)
    
    # penalize the ensemble tracks close to the failed-observation points
    ens_wts = penalize_ensTcks_close_failedObs(ens_wts, ensTracks, dud_obs, forecast_periods, senFPr, r_f)

    return ens_wts

def penalize_ensTcks_close_failedObs(ens_wts, ensTracks, dud_obs, forecast_periods, senFPr, r_f):
    """ Penalize ensemble tracks close to the failed-observation position. """    
    nens = ensTracks.shape[0]
    
    # iterate over all failed observations
    for fobs in dud_obs:
        fobs_t = fobs[2]
        
        # calculate distance between failed observation point and all other ensembles
        for k in range(0,nens):            
            etck_x = list(ensTracks[k][:,2]) 
            etck_y = list(ensTracks[k][:,3])
            etck_t = forecast_periods
            etcki = track_position_akima(etck_x, etck_y, etck_t, fobs_t) # ensemble-track positions at the time of failed-observation
    
            d_ens_obs = np.linalg.norm(np.array(etcki[0:2])-np.array(fobs[0:2]))
            ens_wts[k] = ens_wts[k]*np.exp(d_ens_obs*r_f) # penalize small distances       

    ens_wts = np.array(ens_wts) / np.sum(ens_wts) # normalize the ensemble weights

    return ens_wts
    
    
def optfunc(ens_wts, ensTracks, forecast_periods, known_storm_position):
    ''' Input function to the optimizer implemented in recalc_ens_weights function.
        Note that the optimzation parameters are the potentially non-normalized ensemble weights.
    '''
    ens_wts = np.array(ens_wts) / np.sum(ens_wts)
    
    # calculate total(over time) error between mean track and observed track from 1st storm observation until last storm observation
    mean_track = mean_ensemble(ensTracks, ens_wts)
    mtck = np.hstack((mean_track[:,2:4], np.array(forecast_periods).reshape(-1,1))) # mean track
    
    cost = calculate_error(known_storm_position, mtck)
    
    return cost

def calculate_track_slope(tck, time1, time2):
    ''' time2 > time1, time2 is the forecast time, 
        time1 is the time at which the forecast is made (previous observation time)
    '''
    pos1 = track_position_akima(tck[:,0], tck[:,1], tck[:,2], time1)
    pos2 = track_position_akima(tck[:,0], tck[:,1], tck[:,2], time2)
    dt = pos2[2] - pos1[2]
    vx = (pos2[0] - pos1[0])/dt
    vy = (pos2[1] - pos1[1])/dt
    return[vx, vy]

def calculate_kstp_slope(known_storm_position):
    """ Calculate slope of the track formed by previous known storm center positions."""
    lastIndx = known_storm_position.shape[0]-1
    pos1 = known_storm_position[lastIndx-1]
    pos2 = known_storm_position[lastIndx]
    dt = pos2[2] - pos1[2]
    vx = (pos2[0] - pos1[0])/dt
    vy = (pos2[1] - pos1[1])/dt
    return[vx, vy]

def calculate_error(known_storm_position, tck):
    ''' Calculate average of the errors over between the known storm positions and the input track.
    '''
    if(known_storm_position.size==0):
        return 0
    t_fine = known_storm_position[:,2]
    tck_fine = track_position_akima(tck[:,0], tck[:,1], tck[:,2], t_fine)
    ksps = [known_storm_position[:,0], known_storm_position[:,1], known_storm_position[:,2]]
    error = np.sum(np.linalg.norm(np.array(ksps[0:2])-np.array(tck_fine[0:2])) )
    error = error/known_storm_position.size
    return error

def mid_point_regressed_line(Xsamples, Ysamples, time):
    ''' Mid-point of the regressed line.
    '''
     # Create linear regression object
    regrX = linear_model.LinearRegression()
    regrY = linear_model.LinearRegression()

    # Train the model using the training sets
    regrX.fit(time.reshape(-1,1), Xsamples.reshape(-1,1))
    regrY.fit(time.reshape(-1,1), Ysamples.reshape(-1,1))

    # find known storm position at some point on and in the middle of the regressed line
    tmid = time[0] + 0.5 * (time[-1] - time[0])
    xmid = regrX.predict(tmid.reshape(-1,1)).flatten()
    ymid = regrY.predict(tmid.reshape(-1,1)).flatten()

    return [xmid[0], ymid[0], tmid]

def calculate_track_slope(tck, time1, time2):
    ''' time2 > time1, time2 is the forecast time, 
        time1 is the time at which the forecast is made (previous observation time)
    '''
    pos1 = track_position_akima(tck[:,0], tck[:,1], tck[:,2], time1)
    pos2 = track_position_akima(tck[:,0], tck[:,1], tck[:,2], time2)
    dt = pos2[2] - pos1[2]
    vx = (pos2[0] - pos1[0])/dt
    vy = (pos2[1] - pos1[1])/dt
    return[vx, vy]

def calculate_kstp_slope(known_storm_position):
    """ Calculate the slope of the track formed by the known-storm positions. """
    lastIndx = known_storm_position.shape[0]-1
    pos1 = known_storm_position[lastIndx-1]
    pos2 = known_storm_position[lastIndx]
    dt = pos2[2] - pos1[2]
    vx = (pos2[0] - pos1[0])/dt
    vy = (pos2[1] - pos1[1])/dt
    return[vx, vy]

def process_raw_storm_center_data(raw_storm_position, regr_seg_gap_hrs, regr_subseg_length_hrs, 
                                  segment_num, segment_raw_data_indices):
    """ Process the raw storm center data and get estimated storm center positions. 
        The raw onservation would yield noisy storm center position acquisitions. The return vector has the *estimated* storm center
        positions at the observation times when the storm center had been captured.
        Note that while the length of segment is variable, the length of the subsegment is always fixed.
    """
    est_storm_position = [] # estimated storm positions from the raw storm positions
    segment_data = {} # contains raw storm info of relevant segment
    min_dps_regrss = 1 # minimum number of datapoints required per segment
    
    ''' Decide the segment indices, based on the gap length between the data points. 
        A gap of more than 'regr_seg_gap_hrs' wwill result in start of a new segment.
    '''
    raws =  raw_storm_position[~np.isnan(raw_storm_position).any(axis=1)] # drop nans
    
    last_valid_acq_time = raws[-1][2]    
    #print('regr_seg_gap_hrs: ', regr_seg_gap_hrs)
    if(raws.shape[0]>1):        
        secondlast_valid_acq_time = raws[-2][2]

        if(last_valid_acq_time - secondlast_valid_acq_time >= regr_seg_gap_hrs):
            segment_num = segment_num + 1 # start new segment
            
    if(segment_num == 0):
        segment_raw_data_indices[int(segment_num)] = [0, raws.shape[0]]
    else:
        segment_raw_data_indices[int(segment_num)] = [segment_raw_data_indices[int(segment_num)-1][1], raws.shape[0]]

    #print('---------------------------------------------')
    #print('segment_raw_data_indices')
    #print(segment_raw_data_indices)
    
    ''' Using the segment raw data indices, make estimated (also termed as known) storm interpolation points.
        Create subsegments when the segment is too long and find a regressed line fitting the subsegment.
        The midpoint of the regressed line serves as a interpolation data-point for the storm center.'''
    # iterate over each segment
    for seg_i in range(0,len(segment_raw_data_indices)):
        segment_data = raws[segment_raw_data_indices[seg_i][0]:segment_raw_data_indices[seg_i][1]]
        
        seg_num_data_pts = segment_data.shape[0]
        segment_data_start_time = segment_data[0,2]
        segment_data_end_time = segment_data[seg_num_data_pts-1,2]
        #print('segment_data')
        #print(segment_data) 
        
        if(seg_num_data_pts >= min_dps_regrss): # minimum of 1 data-points needed per segment.                 
            
            segment_len_hrs = segment_data[seg_num_data_pts-1][2] - segment_data[0][2] 
            #print('segment_len_hrs: ', segment_len_hrs)
            
            # break into sub segments of length 'regr_subseg_length_hrs'. Note the use of ceil.
            num_sub_segments = int(np.ceil(segment_len_hrs/regr_subseg_length_hrs))
            #print('num_sub_segments: ', num_sub_segments)
            
            # iterate over all sub-segments except the last one. The last one is treated differently.
            sseg_i = 0

            # note that the last subsegment is treated differently
            for sseg_i in range(0,num_sub_segments-1):
                #subseg_data = np.where((segment_data[:,2]>= sseg_i*segment_data_start_time and segment_data[:,2]< (sseg_i+1)*segment_data_start_time))
                subseg_data = segment_data[ segment_data[:,2]>= segment_data_start_time + sseg_i*regr_subseg_length_hrs]
                subseg_data = subseg_data[ subseg_data[:,2]<= segment_data_start_time + (sseg_i+1)*regr_subseg_length_hrs ]
                #print('subseg_data')
                #print(subseg_data)
                # create a known storm position at the mid-points of a line regressed over the points of the subsegement
                if(subseg_data.shape[0] >= min_dps_regrss):
                    if(subseg_data.shape[0] == 1):
                        est_storm_position.append([subseg_data[:,0], subseg_data[:,1], subseg_data[:,2]])
                    else:
                        [estStmPosX, estStmPosY, estStmPosT] = mid_point_regressed_line(subseg_data[:,0], subseg_data[:,1], subseg_data[:,2])
                        est_storm_position.append([ estStmPosX, estStmPosY, estStmPosT])

            # for the last subsegment, reference is from the end of the segment
            sseg_i = sseg_i + 1
            subseg_data = segment_data[ segment_data[:,2]>= segment_data_end_time - regr_subseg_length_hrs]
            #print('subseg_data')
            #print(subseg_data)
            if(subseg_data.shape[0] >= min_dps_regrss):
                [estStmPosX, estStmPosY, estStmPosT] = mid_point_regressed_line(subseg_data[:,0], subseg_data[:,1], subseg_data[:,2])
                est_storm_position.append([estStmPosX, estStmPosY, estStmPosT])
            
    est_storm_position = np.array(est_storm_position, dtype=object)
    
    #print('est_storm_position')
    #print(est_storm_position)

    return [est_storm_position, segment_num, segment_raw_data_indices]

def algo(ensTracks, bestTrack, aemnTrack, obs_times, senFPr, obsErr, forecast_periods, nens, 
         extrpl_lastKnown_thresh, regr_seg_gap_hrs, regr_subseg_length_hrs, wetp, r_f, extrpl_numKnown_thresh):
    """
    @param ensTracks: Ensemble track data
    
    @param bestTrack: Best track data
    
    @param aemnTrack: Mean ensemble track (AEMN track) data
    
    @param obs_times: Vector of observation times (opportunities) over which the algorithm is iterated.
    
    @param senFPr: sensor footprint radius in meters
    
    @param obsErr: Instrument observation error in [m^2]. Variance (hence [m^2]) from actual position
    
    @param nens: Number of ensembles
    
    @param extrpl_lastKnown_thresh: (Algorithm parameter) Threshold time-period since the last time the storm was seen in hours.  
    
    @param regr_seg_gap_hrs: (Algorithm parameter) Threshold gap between the segments in hours.
    
    @param regr_subseg_length_hrs: (Algorithm parameter) Subsegment length in hours.
    
    @param wetp: (Algorithm parameter) Weight assigned to the slope of the optimal mean ensmble track during extrapolation (1-wetp) is assigned to the slope from the last known storm-positions
    
    @param r_f: (Algorithm parameter) Reward factor (related to penalization factor for penalizing ensemble tracks close to failed observations)
    
    @param extrpl_numKnown_thresh: (Algorithm parameter) number of storm center estimations to be made after which extrapolation can start
    
    """
    storm_radius = 100e3 # [meters] Storm radius 
    timeSinceLastKnown = np.inf
    
    ntst = len(forecast_periods)

    # initialize weigths of the ensembles
    ens_wts = np.ones((nens, 1))
    ens_wts = ens_wts/ np.sum(ens_wts)
    ens_wts_met1 = np.ones((nens, 1))
    ens_wts_met1 = ens_wts/ np.sum(ens_wts)                  

    ens_wts0 = np.ones((nens, 1))
    ens_wts0 = ens_wts0/ np.sum(ens_wts0)
    mean0_track = mean_ensemble(ensTracks, ens_wts0)

    bt_x = list(bestTrack[:,2]) # get best-track info
    bt_y = list(bestTrack[:,3])
    bt_t = forecast_periods

    nobs = len(obs_times)

    obs_positions =  np.full((nobs, 3), np.nan) # list to recored the observed positions
    seen = np.full((nobs, 1), np.nan)  # list of storm seen or not 
    raw_storm_position =  np.full((nobs, 3), np.nan) # note that raw storm position may not be same as the actual storm position due to errors in calculation from the sensed observation data
    raw_storm_position_error =  np.full((nobs, 1), np.nan) 
    known_storm_position =  np.array([]) 
    bt_positions =  np.full((nobs, 3), np.nan) # list to recored the best/ true positions corresponding to the observation times

    segment_raw_data_indices = {}  # contains indices mappting to raw storm info array (with dropped NaNs) 
    segment_num = int(0)
    nksp = 0 # initilize number of known storm positions

    d_obs_bt =  np.full((nobs, 1), np.inf) # list of distances from observation point to best track at the observed times
    d_aemn_bt =  np.full((nobs, 1), np.inf) # list of distances from aemn track positions (~mens0) to best track at the observed times

    # iterate over the observation times
    for indx in range(0,nobs):

        obsti = obs_times[indx] # instant of observation for the current iteration

        ''' calculate observation point based on previous ensemble weights and past storm track if known''' 
        mean_track = mean_ensemble(ensTracks, ens_wts)
        mtck = np.hstack((mean_track[:,2:4], np.array(forecast_periods).reshape(-1,1)))
        mtck_x = list(mean_track[:,2]) 
        mtck_y = list(mean_track[:,3])
        mtck_t = forecast_periods
        mtcki = np.array(track_position_akima(mtck_x, mtck_y, mtck_t, obsti)).flatten()

        currTime = obsti
        if(nksp>0):
            timeSinceLastKnown = currTime - known_storm_position[-1][2]

        if(nksp >= extrpl_numKnown_thresh and timeSinceLastKnown <= extrpl_lastKnown_thresh): #method2
            # extrapolate the Known storm track to the current time
            last_known_position = known_storm_position[known_storm_position.shape[0] -1]
            slope_mtck = (calculate_track_slope(mtck, last_known_position[2], obsti))
            slope_lkstp = (calculate_kstp_slope(known_storm_position))

            dt = obsti - last_known_position[2] 
            dx = (wetp*slope_mtck[0] + (1-wetp)*slope_lkstp[0])*dt
            dy = (wetp*slope_mtck[1] + (1-wetp)*slope_lkstp[1])*dt
            obs_positions[indx][0] = last_known_position[0] + dx
            obs_positions[indx][1] = last_known_position[1] + dy
            obs_positions[indx][2] = obsti    
            
        else:
            obs_positions[indx] = mtcki

        ''' Begin Environmental simulation
            Below part of code is for simulation only. In practise info on the storm being observed or not is obtained from 
            the onboard processor which analyses the obtained image/ observation. 
        '''    
        bt_t = forecast_periods
        bti = track_position_akima(bt_x, bt_y, bt_t, obsti) # get the true storm position at observation time 
        bt_positions[indx] = bti

        # calculate the distance 'd' between the true storm position and the observation point
        d = np.linalg.norm(np.array(obs_positions[indx][0:2])-np.array(bti[0:2])) 
        if(d <  senFPr): 
            seen[indx] = True
            cov =  [[obsErr, 0], [0, obsErr]]  
            x, y = np.random.multivariate_normal(bti[0:2], cov).T
            raw_storm_position[indx][0] = x
            raw_storm_position[indx][1] = y
            raw_storm_position[indx][2] = bti[2]
            raw_storm_position_error[indx] = np.linalg.norm(np.array(raw_storm_position[indx][0:2])-np.array(bti[0:2]))  # log error
            ''' Substitute this snippet to simulate the case of 'no error in storm position acquisition'.
            raw_storm_position[indx] =  np.array(bti[0:3])
            known_storm_position = raw_storm_position
            '''
            ''' Process the raw storm information.
            '''     
            [known_storm_position, segment_num, segment_raw_data_indices] = process_raw_storm_center_data(raw_storm_position, regr_seg_gap_hrs, regr_subseg_length_hrs, 
                                                             segment_num, segment_raw_data_indices)

            nksp = known_storm_position.shape[0]

        else:
            seen[indx] = False
            raw_storm_position[indx] =  np.nan
            raw_storm_position_error[indx] = np.nan
        '''End of environmental simulation '''      

        ''' Recalcuate the ensemble weights from new knowlwedge of the observation made'''
        dud_obs = obs_positions[np.where(seen==0)[0]] # dud observations
        _ens_wts = recalc_ens_weights(known_storm_position, dud_obs, ens_wts, ensTracks, forecast_periods, senFPr, r_f)

        ens_wts = _ens_wts
               
        ''' record metrics wrt to the aemn track to evaluate the algorithm ''' 
        d_obs_bt[indx] = np.linalg.norm(np.array(obs_positions[indx][0:2])-np.array(bti[0:2])) 

        aemn_x = list(mean0_track[:,2]) 
        aemn_y = list(mean0_track[:,3])
        aemn_t = forecast_periods
        aemni = track_position_akima(aemn_x, aemn_y, aemn_t, obsti)
        d_aemn_bt[indx] = np.linalg.norm(np.array(aemni[0:2])-np.array(bti[0:2]))
        
    # pack results to be returned from the algorithm
    results =  { 'obs_positions': obs_positions, # list of observation positions
                 'known_storm_position': known_storm_position, # list of known storm positions (processed from the observations)
                 'error_wna': np.sum(d_obs_bt)/len(d_obs_bt)*1e-3, # average distance-error upon using the EGCTF algorithm
                 'num_succ_cap_wna': np.sum(seen),  # number of successfull captures upon using the EGCTF algorithm
                 'error_waemn' :  np.sum(d_aemn_bt)/len(d_aemn_bt)*1e-3,  # average distance-error upon using the baseline algorithm (AEMN-track only)
                 'num_succ_cap_waemn': sum(d_aemn_bt<senFPr)[0],  # number of successfull captures upon using the baseline algorithm (AEMN-track only)
                 'd_obs_bt': d_obs_bt, # list of distance errors of the captures at the observation times upon using the EGCTF algorithm
                 'd_aemn_bt': d_aemn_bt, # list of distance errors of the captures at the observation times upon using the baseline (AEMN-track only) algorithm
                 'seen': seen, # vector of seen (1) or not-seen (0) at the observation times
                 'bt_positions': bt_positions} # best track positions at the observation times
        
    return results

def summarize_results(ex, aemnTrack, bestTrack, algo_results):
    """ Function which produces plots summarizing the results onto one plot per test-case. """

    ''' ALL RESULTS SUMMARY'''
    %matplotlib notebook
    #calling it a second time may prevent some graphics errors
    %matplotlib notebook  
    import matplotlib.pyplot as plt
    import matplotlib
    levels = [0, 1, 2, 3, 4, 5]
    colors = ['black', 'yellow', 'brown', 'green', 'blue']
    cmap, norm = matplotlib.colors.from_levels_and_colors(levels, colors)

    aemn_x = list(aemnTrack[:,2]) 
    aemn_y = list(aemnTrack[:,3])
    aemn_t = forecast_periods
    aemn_fine_t = np.linspace(0,max(forecast_periods),1000)
    aemn_fine_x, aemn_fine_y, aemn__fine_t = track_position_akima(aemn_x, aemn_y, aemn_t, aemn_fine_t)

    bt_x = list(bestTrack[:,2]) 
    bt_y = list(bestTrack[:,3])
    bt_t = forecast_periods
    bt_fine_t = np.linspace(0,max(forecast_periods),1000)
    bt_fine_x, bt_fine_y, bt_fine_t = track_position_akima(bt_x, bt_y, bt_t, bt_fine_t)


    fig, ((ax1, ax2), (ax6, ax7)) = plt.subplots(nrows=2, ncols=2, figsize=(13, 7), sharex ='row', sharey = 'row')

    ax1.plot(obs_times, algo_results['d_aemn_bt']*1e-3, 'k.', label='AEMN')
    ax1.legend(loc="upper right")
    ax1.text(0.35, 0.9, "%0.2f" % algo_results['error_waemn'], horizontalalignment='center', verticalalignment='center', transform=ax1.transAxes)

    ax2.plot(obs_times, algo_results['d_obs_bt']*1e-3, 'r.', label='ALGO')
    ax2.legend(loc="upper right")
    ax2.text(0.35, 0.9, "%0.2f" % algo_results['error_wna'], horizontalalignment='center', verticalalignment='center', transform=ax2.transAxes)

    ax1.set_ylabel('Error Distance [km]')

    ax1.set_xlabel('Time [hrs]')
    ax2.set_xlabel('Time [hrs]')


    for tn in range(0,nens):
        ax6.plot(ensTracks[tn][:,2]*1e-6, ensTracks[tn][:,3]*1e-6, 'b--')
    ax6.plot(aemnTrack[:,2]*1e-6, aemnTrack[:,3]*1e-6, 'c')
    ax6.plot(bestTrack[:,2]*1e-6, bestTrack[:,3]*1e-6, 'k')
    ax6.text(0.35, 0.9, str(algo_results['num_succ_cap_waemn']), horizontalalignment='center', verticalalignment='center', transform=ax6.transAxes)

    ax7.plot(aemn_fine_x*1e-6, aemn_fine_y*1e-6, 'k--', bt_fine_x*1e-6, bt_fine_y*1e-6, 'k')
    ax7.scatter(algo_results['obs_positions'][:,0]*1e-6, algo_results['obs_positions'][:,1]*1e-6, c = algo_results['seen'].flatten(), marker='s', cmap=cmap, norm=norm,)
    ax7.text(0.35, 0.9, str(algo_results['num_succ_cap_wna']), horizontalalignment='center', verticalalignment='center', transform=ax7.transAxes)

    ax6.set_ylabel('Position-Y 1e6 [m]')

    ax6.set_xlabel('Position-X 1e6 [m]')
    ax7.set_xlabel('Position-X 1e6 [m]')


    fig.suptitle('ex:' + str(ex) + '  Sensor footprint radius:' + str(int(senFPr*1e-3)) + 'km', size=16)
    plt.savefig('sim_data/' + str(ex)+'.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10740094692250186, 'fraction_numerical': 0.017900157820416977, 'mean_word_length': 3.261769911504425, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '448920', 'license': 'MIT', 'n_tokens_mistral': 8311, 'n_tokens_neox': 7997, 'n_words': 2006, 'path': 'jupyter_notebooks/egctf_main.ipynb', 'repo_name': 'vinay0000/ensemble_guided_cyclone_track_forecasting', 'size': '24078'}