<jupyter_start><jupyter_text>Add all traffic to the same graphic<jupyter_code>color_index=0

[bins, bin_centers]  = make_bins(
    lower_bound = 0,
    upper_bound = 400,
    step_size   = 10
)

fig = plt.figure()
ax1 = fig.add_subplot(111)
length = len(bin_centers) 

for vessel in oil_attrs['categories']['all_vessels']:
    
    binned_count = place_into_bins(
        data['length'][vessel], 
        data['count'][vessel], 
        bins
    )
    
    step_size = length_bins[vessel]['step_size']
    ax1.bar(
        bin_centers,
        binned_count,
        label=vessel,
        width=step_size,
        color=colors[color_index]
    )
    
    color_index += 1

plt.xlabel('AIS vessel length (meters)',fontsize=12)
plt.ylabel('Number of ship tracks',fontsize=12)
plt.tight_layout()
plt.show() 
ax1.legend()<jupyter_output><empty_output><jupyter_text>Scale ATB by time interval to see if/how it affects resultI still need to check that the result that all are scaling down is a good result and not a reflection that I made a mistake<jupyter_code># create bins for both raw and scaled data
[bins, bin_centers]  = make_bins(
    lower_bound = length_bins['atb']['lower_bound'],
    upper_bound = length_bins['atb']['upper_bound'],
    step_size   = length_bins['atb']['step_size']
)

# bin the raw data first
binned_count = place_into_bins(
    data['length']['atb'], 
    data['count']['atb'], 
    bins
)

atb_bin_centers = bin_centers
atb_binned_count = binned_count

# scale tracks according to time interval between pings
scaling_factor = (atb_data_df['Median time between pings (minutes)']/
                  numpy.mean(atb_data_df['Median time between pings (minutes)'])
                 )
scaling_factor_median = (atb_data_df['Median time between pings (minutes)']/
                  numpy.median(atb_data_df['Median time between pings (minutes)'])
                 )
    
atb_tracks = (atb_data_df['COUNT'] * scaling_factor)
atb_tracks_median = (atb_data_df['COUNT'] * scaling_factor_median)

# place scaled tracks into length bins
binned_tracks = place_into_bins(
    atb_data_df['LENGTH'], 
    atb_tracks, 
    bins
)

binned_tracks_median = place_into_bins(
    atb_data_df['LENGTH'], 
    atb_tracks_median, 
    bins
)

fs = 14
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111)    # The big subplot
ax1 = fig.add_subplot(311)
ax2 = fig.add_subplot(312)
ax3 = fig.add_subplot(313)

ax1.bar(
    atb_bin_centers,
    atb_binned_count,
    width=15,
    label='Raw AIS count',
    color='darkslateblue'
)
ax1.bar(
    bin_centers,
    binned_tracks,
    width=10,
    label='Mean-scaled count',
    color='yellowgreen'
)
ax1.bar(
    bin_centers,
    binned_tracks_median,
    width=5,
    label='Median-scaled count',
    color='gold'
)
# scatter of raw data
ax1.scatter(
    data['length']['atb'],
    data['count']['atb'],
    label='Raw AIS count',
    color='slateblue'
)
# scatter of scaled data
ax1.scatter(
    data['length']['atb'],
    atb_tracks,
    label='Scaled AIS count',
    color='olivedrab',
    edgecolor='yellowgreen'
)
#  Add scatter to second graphis to see it better
# scatter of raw data
ax2.scatter(
    data['length']['atb'],
    data['count']['atb'],
    label='Raw AIS count',
    color='slateblue'
)
# scatter of scaled data
ax2.scatter(
    data['length']['atb'],
    atb_tracks,
    label='Scaled AIS count',
    color='olivedrab',
    edgecolor='yellowgreen'
)

# plot the scaling factor
# scatter of scaled data
ax3.scatter(
    data['length']['atb'],
    scaling_factor,
    color='dimgrey',
    label='mean scaling'
)
ax3.scatter(
    data['length']['atb'],
    scaling_factor_median,
    color='black',
    label='median scaling'
)
ax3.plot(
    data['length']['atb'],
    numpy.ones(len(data['length']['atb'])),
    color='lightgrey'
)

# Turn off axis lines and ticks of the big subplot
ax.spines['top'].set_color('none')
ax.spines['bottom'].set_color('none')
ax.spines['left'].set_color('none')
ax.spines['right'].set_color('none')
ax.tick_params(labelcolor='w', top=False, bottom=False, left=False, right=False)

ax1.set_xticklabels('')
ax1.set_ylabel('AIS count', fontsize=fs)
ax1.legend()

ax2.set_ylabel('AIS count', fontsize=fs)
ax2.set_xticklabels('')

ax3.set_ylabel('Scaling factor', fontsize=fs)
ax3.set_xlabel('Vessel lenght (m)', fontsize=fs)
ax3.legend()
plt.show()
test = atb_data_df['Median time between pings (minutes)']/numpy.mean(atb_data_df['Median time between pings (minutes)'])
test<jupyter_output><empty_output><jupyter_text>Show AIS pings multiplied by oil capacity<jupyter_code>from get_oil_capacity import get_oil_capacity

random_seed=None
# Initialize PCG-64 random number generator
random_generator = numpy.random.default_rng(random_seed)
fuel = {}
cargo={}

# create lengths for calculating oil capacity
[bins, bin_centers]  = make_bins(
    lower_bound = 0,
    upper_bound = 400,
    step_size   = 10
)

# calculate oil capacity by length and vessel type
for vessel in oil_attrs['categories']['all_vessels']:
    for length in bin_centers:
        [fuel_output,cargo_output] = get_oil_capacity(
            input_file, 
            length, 
            vessel, 
            random_generator
        )
        
        if vessel not in fuel:
            fuel[vessel]=[fuel_output]
            cargo[vessel]=[cargo_output]
        else:
            fuel[vessel].append(fuel_output)
            cargo[vessel].append(cargo_output)<jupyter_output><empty_output><jupyter_text>Calculated scaled values for AIS count<jupyter_code>binned_count = {}
binned_count['raw'] = {}
binned_count['fuel_scaled'] = {}
binned_count['cargo_scaled'] = {}

for vessel in oil_attrs['categories']['all_vessels']:
    
    binned_data = place_into_bins(
        data['length'][vessel], 
        data['count'][vessel], 
        bins
    )
    
    # store values
    binned_count['raw'][vessel] = binned_data
    
    # multiply counts in length bins by oil capacity in bin
    binned_count['fuel_scaled'][vessel] = binned_data * fuel[vessel] 
    binned_count['cargo_scaled'][vessel] = binned_data * cargo[vessel]
    
    # correct barge traffic to reflect 1 oil cargo barge for every ~10 barges. 
    if 'barge' in vessel:
        binned_count['cargo_scaled'][vessel] = binned_data * cargo[vessel]/10<jupyter_output><empty_output><jupyter_text>First plot by fuel oil capacity and length<jupyter_code>from textwrap import wrap
fs = 14
fig = plt.figure(figsize=(10,15))
ax = fig.add_subplot(111)    # The big subplot
ax1 = fig.add_subplot(311)
ax2 = fig.add_subplot(312)
ax3 = fig.add_subplot(313)

length = len(bin_centers) 
color_index=0
for vessel in oil_attrs['categories']['all_vessels']:
    ax1.bar(
        bin_centers,
        binned_count['raw'][vessel],
        label=vessel,
        width = step_size,
        color=colors[color_index]
    )
    
    ax2.bar(
        bin_centers,
        binned_count['fuel_scaled'][vessel],
        label=vessel,
        width = step_size,
        color=colors[color_index]
    )
    
    ax3.bar(
        bin_centers,
        binned_count['cargo_scaled'][vessel],
        label=vessel,
        width = step_size,
        color=colors[color_index]
    )
    color_index += 1
    
# Turn off axis lines and ticks of the big subplot
ax.spines['top'].set_color('none')
ax.spines['bottom'].set_color('none')
ax.spines['left'].set_color('none')
ax.spines['right'].set_color('none')
ax.tick_params(labelcolor='w', top=False, bottom=False, left=False, right=False)

# Set common labels
# ax.set_xlabel('common xlabel')
# ax.set_ylabel('common ylabel')

# Set specific plot labels
ax1.set_ylabel('Raw AIS count',fontsize=fs)
ax2.set_ylabel('Scaled by fuel capacity',fontsize=fs)
ax3.set_ylabel('Scaled by cargo capacity',fontsize=fs)
ax1.set_xticklabels('')
ax2.set_xticklabels('')

ax3.set_xlabel('AIS vessel length (meters)',fontsize=fs)

ax1.legend()
ax2.legend()
ax3.legend()
# plt.tight_layout()
# plt.show() 
# ax1.legend()<jupyter_output><empty_output><jupyter_text>Plot ship tracks by vessel type and length<jupyter_code>color_index=0
for vessel in oil_attrs['categories']['all_vessels']:

    lower_bound = length_bins[vessel]['lower_bound']
    upper_bound = length_bins[vessel]['upper_bound']
    step_size   = length_bins[vessel]['step_size']

    [bins, bin_centers]  = make_bins(
        lower_bound,
        upper_bound,
        step_size
    )

    binned_count = place_into_bins(
        data['length'][vessel], 
        data['count'][vessel], 
        bins
    )
    
    fig = plt.figure()
    ax1 = fig.add_subplot(111)
    ax1.bar(
        bin_centers,
        binned_count,
        label=vessel,
        width = step_size/2,
        color=colors[color_index]
    )
    plt.xlabel('AIS vessel length (meters)',fontsize=12)
    plt.ylabel('Number of ship tracks',fontsize=12)
    plt.tight_layout()
    ax1.legend()
    plt.show() 
    
    color_index += 1
mins = numpy.zeros(len(oil_attrs['categories']['all_vessels']))
maxs = numpy.zeros(len(oil_attrs['categories']['all_vessels']))
i = 0

data['length']['ferry'].replace(numpy.nan,0)
for vessel in oil_attrs['categories']['all_vessels']:
    if vessel == 'ferry':
        not_nan = ~numpy.isnan(data["length"]['ferry'])
        real_ferry = data["length"]['ferry'][not_nan]
        print(f'{vessel}[ min: {min(real_ferry)}, max: {max(real_ferry)}]')
        mins[i] = min(real_ferry)
        maxs[i] = max(real_ferry)
    else:
        print(f'{vessel}[ min: {min(data["length"][vessel])}, max: {max(data["length"][vessel])}]')
        mins[i] = min(data["length"][vessel])
        maxs[i] = max(data["length"][vessel])
    i+=1
    
# take a look at oil capacity and fuel capacity values

oil_attribution_dir  = pathlib.Path(
    '/Users/rmueller/Data/MIDOSS/marine_transport_data'
)
oil_attribution_file = oil_attribution_dir/'oil_attribution.yaml'

min_fuel_capacity = numpy.zeros(len(mins))
max_fuel_capacity = numpy.zeros(len(mins))
min_cargo_capacity = numpy.zeros(len(mins))
max_cargo_capacity = numpy.zeros(len(mins))

for i in range(len(mins)):

    min_vessel_len, max_vessel_len, vessel_type = mins[i], maxs[i], oil_attrs['categories']['all_vessels'][i]
    min_fuel_capacity[i], min_cargo_capacity[i] = get_oil_capacity(
            oil_attribution_file, min_vessel_len, vessel_type, random_generator
        )
    max_fuel_capacity[i], max_cargo_capacity[i] = get_oil_capacity(
            oil_attribution_file, max_vessel_len, vessel_type, random_generator
        )<jupyter_output><empty_output><jupyter_text>compare calculated fuel capacities with limits<jupyter_code>min_fuel_boundary  = []
max_fuel_boundary  = []
min_cargo_boundary = []
max_cargo_boundary = []
for vessel in oil_attrs['categories']['all_vessels']:
    min_fuel_boundary.append(oil_attrs['vessel_attributes'][vessel]['min_fuel'])
    max_fuel_boundary.append(oil_attrs['vessel_attributes'][vessel]['max_fuel'])
    if vessel in oil_attrs['categories']['tank_vessels']:
        min_cargo_boundary.append(oil_attrs['vessel_attributes'][vessel]['min_cargo'])
        max_cargo_boundary.append(oil_attrs['vessel_attributes'][vessel]['max_cargo'])
    else:
        min_cargo_boundary.append(numpy.NaN)
        max_cargo_boundary.append(numpy.NaN)
len(min_cargo_boundary)
data = {'vessel type': oil_attrs['categories']['all_vessels'],
        'min observed length': mins,
        'min get_fuel_capacity'  : min_fuel_capacity,
        'min fuel threshold'     : min_fuel_boundary, 
        'min get_cargo_capacity' : min_cargo_capacity,
        'min cargo threshold'    : min_cargo_boundary,
        'max observed length': maxs,
        'max get_fuel_capacity'  : max_fuel_capacity,
        'max fuel threshold'     : max_fuel_boundary, 
        'max get_cargo_capacity' : max_cargo_capacity,
        'max cargo threshold'    : max_cargo_boundary, 
       }
df = pandas.DataFrame (data, columns = list(data.keys()))
df
### test upper and lower boundaries

for i in range(len(oil_attrs['categories']['all_vessels'])):

    min_fuel_capacity[i], min_cargo_capacity[i] = get_oil_capacity(
            oil_attribution_file, 0.1, oil_attrs['categories']['all_vessels'][i], random_generator
        )
    max_fuel_capacity[i], max_cargo_capacity[i] = get_oil_capacity(
            oil_attribution_file, 10000, oil_attrs['categories']['all_vessels'][i], random_generator
        )
   
data = {'vessel type': oil_attrs['categories']['all_vessels'],
        'min observed length': mins,
        'min get_fuel_capacity'  : min_fuel_capacity,
        'min fuel threshold'     : min_fuel_boundary, 
        'min get_cargo_capacity' : min_cargo_capacity,
        'min cargo threshold'    : min_cargo_boundary,
        'max observed length': maxs,
        'max get_fuel_capacity'  : max_fuel_capacity,
        'max fuel threshold'     : max_fuel_boundary, 
        'max get_cargo_capacity' : max_cargo_capacity,
        'max cargo threshold'    : max_cargo_boundary, 
       }
df = pandas.DataFrame (data, columns = list(data.keys()))
df
# import sys 
# sys.path.append('/usr/lib/python3/dist-packages')
# df.to_excel('fuel_and_cargo_thresholds.xlsx')
# fig, ax = plt.subplots(1)
# ax.plot(
#     oil_attrs['categories']['all_vessels'], 
#     min_fuel_capacity,
#     'b')
# ax.fill_between(
#     oil_attrs['categories']['all_vessels'], 
#     oil_attrs['vessel_attributes'][:]['min_fuel'], 
#     oil_attrs['vessel_attributes'][:]['max_fuel'],
#     facecolor='blue', alpha=0.5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13222403803298172, 'fraction_numerical': 0.010251077105927797, 'mean_word_length': 3.8991994177583695, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '16968', 'license': 'Apache-2.0', 'n_tokens_mistral': 5004, 'n_tokens_neox': 4710, 'n_words': 859, 'path': 'notebooks/monte_carlo_dev/plot_AIScount_by_vessel_length.ipynb', 'repo_name': 'MIDOSS/analysis-rachael', 'size': '13462'}