<jupyter_start><jupyter_text>Create ATB oil capacity weights by  AIS ship tracks by:1. not-scaled values and2. values scaled by median ping intervals<jupyter_code>import sys
import yaml
import pathlib
import numpy
import pandas
import matplotlib.pyplot as plt
from monte_carlo_utils import make_bins, get_bin, place_into_bins
input_directory = pathlib.Path(
    '/Users/rmueller/Data/MIDOSS/marine_transport_data/'
)

### open master yaml file for 
with open(input_directory/'master.yaml') as file:
    master = yaml.load(file, Loader=yaml.Loader)
    
#~~~ atb traffic ~~~
# C: "AIS ship track frequency"
# D: "Median ship track distance (km)"
# K: "Vessel length (AIS)"
# L: "Vessel Length (AIS + <50 m correction)"
# O: "Total Fuel Capacity (liters)"
# P: "Cargo Capacity (liters)"

atb_df = pandas.read_excel(
    master['files']['oil_capacity'],
    sheet_name='atb', 
    usecols='A,C,D,K,L,O,P',
    header=1
)
atb_data_df = pandas.read_excel(
    master['files']['oil_capacity'],
    sheet_name='atb data', 
    usecols='A,B,D,E'
)
for ind in range(len(atb_data_df['MMSI'])):
    print(ind,atb_data_df['MMSI'][ind], atb_df['MMSI'][ind], atb_df['AIS ship tracks'][ind], atb_data_df['COUNT'][ind])
# create line fit for atbs with length > 50 m
# fit = coefficient[1] + coefficient[0] * length
good_values = atb_df['Vessel Length (AIS + <50 m correction)']>0

# create fuel and cargo capacity bins into which vessel traffic information 
# is sorted to determine oil capacity weights. 
[cargo_bins, cargo_bin_centers]  = make_bins(
    lower_bound =  4000000,
    upper_bound = 30000000,
    step_size   = 2000000
)

# ~~~~~~~~~~~~~~~~~~~~ ORIGINAL WAY ~~~~~~~~~~~~~~~~~~~~~~~~
# we use this estimate of distance traveled by each atb in 2018 to weight
# oil capacity allocation for vessels that don't have a clear allocation 
atb_distance = (atb_df['AIS ship tracks'] * 
     atb_df['Median ship track distance (km)'])

### cargo weights ###
# bin distance travelled by cargo capacity and sum distances for each 
# cargo capacity bin to create cargo capacity weights
binned_atb_distance_by_cargo = place_into_bins(
    atb_df['Cargo Capacity (liters)'], 
    atb_distance, 
    cargo_bins
)


# ~~~~~~~~~~~~~~~~~~~~ NEW WAY ~~~~~~~~~~~~~~~~~~~~~~~~
# we use this estimate of distance traveled by each atb in 2018 to weight
# oil capacity allocation for vessels that don't have a clear allocation 
atb_tracks = (atb_df['AIS ship tracks'] * 
     atb_data_df['Median time between pings (minutes)']/
     numpy.mean(atb_data_df['Median time between pings (minutes)'])
)

### cargo weights ###
# bin distance travelled by cargo capacity and sum distances for each 
# cargo capacity bin to create cargo capacity weights
binned_atb_tracks_by_cargo = place_into_bins(
    atb_df['Cargo Capacity (liters)'], 
    atb_tracks, 
    cargo_bins
)<jupyter_output><empty_output><jupyter_text>Plot comparison of old vs. new way<jupyter_code>#~~~~ PLOT ship tracks by time interval ~~~~~~~~~~~~~~~~~~~~~
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.scatter(
    atb_data_df['COUNT'],
    atb_data_df['Median time between pings (minutes)'],
    50,
    label='ship track interval',
    color='grey'
)
plt.xlabel('Total number of AIS pings',fontsize=12)
plt.ylabel('Median time interval (minutes)',fontsize=12)
plt.tight_layout()
ax1.legend()
plt.show() 

#~~~~ PLOT ship tracks by time interval (with errorbars) ~~~~
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.errorbar(
    atb_data_df['COUNT'],
    atb_data_df['Median time between pings (minutes)'],
    yerr=atb_data_df['Standard deviation (minutes)']/2,
    fmt='o', 
    color='grey',
    ecolor='lightgray', 
    elinewidth=3, 
    capsize=0,
    ls='none'
)
# ax1.scatter(
#     atb_data_df['COUNT'],
#     atb_data_df['Median time between pings (minutes)'],
#     50,
#     label='ship track interval',
#     color='grey'
# )
plt.xlabel('Total number of AIS pings',fontsize=12)
plt.ylabel('Median time interval (minutes)',fontsize=12)
plt.tight_layout()
ax1.legend()
plt.show() 

#~~~~ PLOT ORIGINAL METHOD ~~~~
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.bar(
    cargo_bin_centers,
    binned_atb_distance_by_cargo,
    width=0.3e6,
    label='Binned voyage lengths',
    color='darkslateblue'
)
ax1.scatter(
    atb_df['Cargo Capacity (liters)'],
    atb_distance,
    50,
    label='DOE oil transfers',
    color='yellowgreen'
)
plt.xlabel('Oil cargo capacity (liters)',fontsize=12)
plt.ylabel('Distance traveled by tugs',fontsize=12)
plt.tight_layout()
ax1.legend()
plt.show() 

#~~~~ PLOT NEW METHOD ~~~~
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.bar(
    cargo_bin_centers,
    binned_atb_tracks_by_cargo,
    width=0.3e6,
    label='Binned AIS tracks',
    color='darkslateblue'
)
ax1.scatter(
    atb_df['Cargo Capacity (liters)'],
    atb_tracks,
    50,
    label='DOE oil transfers',
    color='yellowgreen'
)
plt.xlabel('Oil cargo capacity (liters)',fontsize=12)
plt.ylabel('AIS ship tracks',fontsize=12)
plt.tight_layout()
ax1.legend()
plt.show() 

#~~~~ PLOT oil cargo weighting ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
old_weight = binned_atb_distance_by_cargo/sum(binned_atb_distance_by_cargo)
new_weight = binned_atb_tracks_by_cargo/sum(binned_atb_tracks_by_cargo)

old_weight[numpy.where(old_weight==0)[0]]=numpy.nan
new_weight[numpy.where(new_weight==0)[0]]=numpy.nan

fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.scatter(
    cargo_bin_centers,
    old_weight,
    50,
    label='OLD atb cargo weights',
    color='grey'
)
ax1.scatter(
    cargo_bin_centers,
    new_weight,
    50,
    label='NEW atb cargo weights',
    color='magenta'
)
plt.xlabel('Oil cargo capacity (liters)',fontsize=12)
plt.ylabel('ATB cargo weighting',fontsize=12)
plt.tight_layout()
ax1.legend()
plt.show()


atb_data_df
sum(old_weight)
sum(new_weight)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1493584260051326, 'fraction_numerical': 0.020701454234388367, 'mean_word_length': 4.84015984015984, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '141104', 'license': 'Apache-2.0', 'n_tokens_mistral': 2249, 'n_tokens_neox': 2098, 'n_words': 497, 'path': 'notebooks/monte_carlo_dev/Plot_ATB_capacity_weights.ipynb', 'repo_name': 'MIDOSS/analysis-rachael', 'size': '5845'}