<jupyter_start><jupyter_text>First Look At Calculating Bunches & Gaps  Helper Functions<jupyter_code>def fcc_projection(loc1, loc2):
    """
    function to apply FCC recommended formulae
    for calculating distances on earth projected to a plane
    
    significantly faster computationally, negligible loss in accuracy
    
    Args: 
    loc1 - a tuple of lat/lon
    loc2 - a tuple of lat/lon
    """
    lat1, lat2 = loc1[0], loc2[0]
    lon1, lon2 = loc1[1], loc2[1]
    
    mean_lat = (lat1+lat2)/2
    delta_lat = lat2 - lat1
    delta_lon = lon2 - lon1
    
    k1 = 111.13209 - 0.56605*cos(2*mean_lat) + .0012*cos(4*mean_lat)
    k2 = 111.41513*cos(mean_lat) - 0.09455*cos(3*mean_lat) + 0.00012*cos(5*mean_lat)
    
    distance = sqrt((k1*delta_lat)**2 + (k2*delta_lon)**2)
    
    return distance
def get_dwell(df):    
    dwell_count = 0
    dwell_totals = []

    for x in df['kph']:
        if x <= 0:
          dwell_count += 1
          dwell_totals.append(dwell_count)
        elif x > 0:
          dwell_totals.append(0)
          dwell_count = 0
            
    df['dwell'] = [dwell_totals[x] for x in range(len(df))]

    return df
def adjust_timestamp(df):
    times = df.timestamp.values
    ages = df.age.values

    df['adjusted_timestamp'] = [pd.Timestamp(times[x]) - 
                                pd.Timedelta(seconds=ages[x]) 
                                for x in range(len(df.timestamp))]
    
    return df
def assign_stops(df, stops):
    """
    applies basic wrangling function
    calculates nearest stop from reported location in km
    returns dataframe with reported location, 
    nearest stop (coords and name), and distance between
    
    implements FCC projection formulae for calculating distance
    
    Args:
    df - dataframe of transit data, requires 'latitude', 'longitude' columns
    stops - datafram of stops data, requires 'lat', 'lon', 'title' columns
    """
    
    stop_lats = stops['lat'].values
    stop_lons = stops['lon'].values

    reported_lats = df['latitude'].values
    reported_lons = df['longitude'].values

    stop_points = [{'latitude': stop_lats[x], 'longitude': stop_lons[x]} 
                 for x in range(len(stops))]

    reported_points = [{'latitude': reported_lats[x], 
                      'longitude': reported_lons[x]} 
                     for x in range(len(df))]

    radius = .1524

    stop_point_tuples = [tuple(stop_points[x].values()) 
                       for x in range(len(stop_points))]

    reported_point_tuples = [tuple(reported_points[x].values()) 
                           for x in range(len(reported_points))]

    df['reported_location'] = reported_point_tuples
        
    distances = [{x: fcc_projection(location, x) 
                 for x in stop_point_tuples} 
                 for location in reported_point_tuples]
        
    distances_sorted = [{k: v for k, v in sorted(distances[x].items(), 
                                                 key=itemgetter(1))}
                       for x in range(len(distances))]
        
    point_stops = [next(iter(distances_sorted[x].items())) 
                   if next(iter(distances_sorted[x].items()))[1] <= radius 
                   else None
                   for x in range(len(distances_sorted))]
 
    stop_tuples = list(zip(stops.lat, stops.lon))
    stop_titles = [stops.title.iloc[stop_tuples.index(stop[0])] 
                   if stop != None
                   else None 
                   for stop in point_stops]
        
    df['nearest_stop'] = [x[0] if x != None else None for x in point_stops]
    df['distance_in_km'] = [x[1] if x != None else None for x in point_stops]

    df['title'] = stop_titles

    df = df.drop(columns=['age', 'rid', 'latitude', 'longitude', 'timestamp',
                          'heading'])
    
    return df
def bin_from_timestamp(timestamp, bins):
    diffs = [abs(timestamp - x) for x in bins]
    return bins[diffs.index(min(diffs))]<jupyter_output><empty_output><jupyter_text>Function To Calculate B&Gs<jupyter_code>def find_disruptions(stops, daily_json, schedules_json):
    
    start = time()

    # pulling relevant info from schedules_json
    route = schedules_json['route']
    intervals = schedules_json['intervals']
    stop_list = schedules_json['stops']

    # filtering by route    
    locations = pd.DataFrame(data=daily_json).dropna()
    locations = locations[locations.rid.eq(route)]
    
    # unifying direction tags for later merging
    locations['direction'] = ['Inbound' if '_I' in x 
                              else 'Outbound' if '_O' in x
                              else None
                              for x in locations.direction.values]
    
    # filtering by route and current active stops
    stops = stops[stops.route_id.eq(route)]
    stops = stops[stops.tag.isin(stop_list)]
    stops = stops.rename(columns={'dir': 'direction'})

    print(f'Prep Complete: {time()-start} Seconds')
    
    # adjusting timestamps
    locations = adjust_timestamp(locations)

    print(f'Timestamps Adjusted: {time()-start} Seconds')

    # calculating dwell time for filtering
    locations = get_dwell(locations)

    print(f'Dwell Calculated: {time()-start} Seconds')
    
    # calculating nearest stops and distances
    locations = assign_stops(locations, stops)

    print(f'Stops Assigned: {time()-start} Seconds')
    
    # merging to assign titles to stops and unify tags
    locations = locations.merge(stops[stops.route_id.eq(route)], 
                                how='left', on=['direction', 'title'])
    
    locations = locations.drop(columns=[
                                        'route_id', 'lat', 
                                        'lon', 'stopId']).dropna()

    # dropping all reports with dwell != 0
    locations = locations[locations.dwell.eq(0)
                          ].reset_index(drop=True
                                        ).sort_values('adjusted_timestamp')
    
    print(f'Stops Filtered: {time()-start} Seconds')

    # separating by stop to calculate service deltas and disruptions
    stop_dfs = []
    for x in locations.tag.unique():
      
        df = locations[locations.tag.eq(x)].copy().reset_index(drop=True)

        df['window'] = [round((locations.loc[x, 'distance_in_km'] / 
                                      np.mean(df.kph))*3600, 2) 
                              for x in range(len(df))]

        df['service_deltas'] = [pd.Timestamp(df.loc[x, 'adjusted_timestamp']) 
                                            - pd.Timestamp(df
                                            .loc[x-1,  'adjusted_timestamp'])
                                            if x != 0
                                            else None
                                            for x in range(len(df))]
        
        df['adjusted_service_deltas'] = [(df.loc[x, 'service_deltas'].total_seconds()
                                        - pd.Timedelta(seconds=df.loc[x, 'window']).total_seconds(),
                                        df.loc[x, 'service_deltas'].total_seconds()
                                        + pd.Timedelta(seconds=df.loc[x, 'window']).total_seconds())
                                        for x in range(len(df))]
        
        df['service_disruptions'] = [1 if (x[0]/60) < .2 * intervals['mode']
                                            else -1 if (x[1]/60) > 1.5 * intervals['mode']
                                            else 0
                                            for x in df['adjusted_service_deltas']]
        
        # dropping instances of multiple identical reports
        df.drop_duplicates(subset=['vid', 'adjusted_timestamp'], 
                          keep='first', inplace=True)
        
        # creating new dataframe of timestamp / stop: service_disruptions
        df = pd.DataFrame(data={'adjusted_timestamp': df.adjusted_timestamp.values,
                                f'{x}': df.service_disruptions.values})

        stop_dfs.append(df)
    
    print(f'Services Analyzed: {time()-start} Seconds')
    
    # merging all stop-level dataframes
    # result is df of columns: stops, rows: timestamps, cells: B&Gs
    daily_service = reduce(lambda x, y: pd.merge(x, y, 
                                               on='adjusted_timestamp', 
                                               how='outer', sort=True), 
                                               stop_dfs)
    
    print(f'Master DataFrame Compiled: {time()-start} Seconds')

    # Forward fill of service                               
    daily_service.fillna(method='ffill', inplace=True, axis=0)
    
    print(f'Naive Service Imputation Complete: {time()-start} Seconds')

    # creating a simple filter for calculating metrics
    stop_columns = daily_service.copy().columns
    
    # calculating basic metrics
    daily_service['on_time'] = daily_service[stop_columns].eq(0).sum(axis=1)
    daily_service['bunches'] = daily_service[stop_columns].eq(1).sum(axis=1)
    daily_service['gaps'] = daily_service[stop_columns].eq(-1).sum(axis=1)
    daily_service['service_disruptions'] = daily_service[['bunches', 
                                                            'gaps']].sum(axis=1)

    daily_service['coverage'] = [round((x['on_time'] + x['bunches']) / 
                                           (x['bunches'] + x['on_time'] 
                                            + x['gaps']
                                            ) * 100, 2) 
                                  for _, x in daily_service.iterrows()]
    
    daily_service['on_time_%'] = [round(x['on_time'] / 
                                            (x['on_time'] + 
                                            x['service_disruptions']
                                             ) * 100, 2)
                                  for _, x in daily_service.iterrows()]
    
    average_coverage = np.mean(daily_service.coverage)
    average_on_time = np.mean(daily_service['on_time_%'])
    total_bunches = daily_service.bunches.sum()
    total_gaps = daily_service.gaps.sum()
    total_disruptions = daily_service.service_disruptions.sum()
    total_on_time = daily_service.on_time.sum()
    total_services = (daily_service.on_time.sum() +
                            daily_service.service_disruptions.sum())

    
    print(f'Metrics Calculated - Total Time: {time()-start} Seconds')
            
    return {'dataframe': daily_service, 
            'metrics': {'coverage': average_coverage,
                        'on_time_%': average_on_time,
                        'totals': {'bunches': total_bunches,
                                   'gaps': total_gaps,
                                   'disruptions': total_disruptions,
                                   'on_time': total_on_time,
                                   'services': total_services}}}
output = find_disruptions(stops, daily_json, sched_json)
daily_service = output['dataframe']
output['metrics']
daily_service.head(5)
fig, ax = plt.subplots()

sns.lineplot(x=daily_service.index, y=daily_service['on_time_%'])
sns.lineplot(x=daily_service.index, y=daily_service.coverage)

ax.legend(['On-Time %', 'Coverage %'])
ax.grid()
fig.set_size_inches(20, 10)
plt.title('On-Time % & Coverage', {'fontsize': 20})
plt.ylabel('Percentage')
plt.xlabel('Time')
plt.show()

fig, ax = plt.subplots()

sns.lineplot(x=daily_service.index, y=daily_service.bunches)
sns.lineplot(x=daily_service.index, y=daily_service.gaps)

ax.legend(['Bunches', 'Gaps'])
ax.grid()
fig.set_size_inches(20, 10)
plt.title('Bunches & Gaps', {'fontsize': 20})
plt.ylabel('Disruptions')
plt.xlabel('Time')
plt.show()

fig, ax = plt.subplots()

sns.lineplot(x=daily_service.index, y=daily_service.on_time)
sns.lineplot(x=daily_service.index, y=daily_service.service_disruptions)

ax.legend(['On-Time', 'Disruptions'])
ax.grid()
fig.set_size_inches(20, 10)
plt.title('On-Time Services & Service Disruptions', {'fontsize': 20})
plt.ylabel('Services')
plt.xlabel('Time')
plt.show();
sns.pairplot(data=daily_service[['on_time_%', 'on_time', 'bunches', 
                               'gaps', 'service_disruptions', 'coverage']]);
sns.jointplot(daily_service.bunches, 
              daily_service.gaps, 
              kind='kde')

sns.jointplot(daily_service.gaps, 
              daily_service['on_time_%'], 
              kind='kde')

sns.jointplot(daily_service.service_disruptions, 
              daily_service['on_time_%'], 
              kind='kde');<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11871964328370094, 'fraction_numerical': 0.011306632693685804, 'mean_word_length': 1.9435200374970705, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 61, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '15388', 'license': 'MIT', 'n_tokens_mistral': 3884, 'n_tokens_neox': 3643, 'n_words': 746, 'path': 'labs24_notebooks/bunches_and_gaps/B&G_FirstLook.ipynb', 'repo_name': 'Lambda-School-Labs/sftma-data-analysis-ds', 'size': '12559'}