<jupyter_start><jupyter_text>Preprocessing  Missing riskSome port calls miss an entry for the risk. Virtually all portcalls with a missing risk, have priority PI.This priority is commonly associated with risk 1 and hence we categorize all these portcalls with risk 1.<jupyter_code>portcalls_cleaned['risk'].isna().value_counts()
portcalls_cleaned.groupby(['priority'])['risk'].value_counts(normalize=True).sort_index()
portcalls_cleaned.loc[lambda x: x['risk'].isna()]['priority'].value_counts()
portcalls_cleaned.fillna({'risk': 1}, inplace=True)<jupyter_output><empty_output><jupyter_text>Filter on dates<jupyter_code>fig, ax = plt.subplots(figsize=(4.82,3.21), constrained_layout=True)
portcalls_cleaned['arrival'].value_counts().resample('M').size().plot(label='portcalls', ax=ax)
inspections_cleaned['DateOfFirstVisit'].value_counts().resample('M').size().plot(label='inspections', ax=ax)

ax.axvline(src.START_DATE, 0, 10_000, color='C2')
ax.axvline(src.END_DATE, 0, 10_000, color='C2')
ax.xaxis.set_ticks_position('both')
ax.yaxis.set_ticks_position('both')

ax.set_ylabel('Counts per month')
ax.set_yscale('log')
ax.legend(loc='lower right')<jupyter_output><empty_output><jupyter_text>Age of ship<jupyter_code>fig, ax = plt.subplots()

sns.ecdfplot(data=portcalls_cleaned.astype({'risk': 'int'}), x='keel_laying', hue='risk', ax=ax)
ax.axvline(portcalls_cleaned['keel_laying'].max() - pd.DateOffset(years=12), c='black', ls=':')<jupyter_output><empty_output><jupyter_text>Type of ship<jupyter_code>df = portcalls_cleaned[['type', 'type_high_risk']].drop_duplicates()
high_risk = df.loc[lambda x: x['type_high_risk'], "type"].values
low_risk = df.loc[lambda x: ~x['type_high_risk'], "type"].values
print(f"High risk types: {high_risk}")
print(f"Low risk types: {low_risk}")<jupyter_output>High risk types: ['Ro-Ro passenger ship' 'Oil tanker' 'Chemical tanker'
 'High speed passenger craft' 'Bulk carrier' 'Passenger ship' 'NLS tanker'
 'Gas carrier' 'Oil tanker/Chemical tanker ' 'Combination carrier'
 'Oil tanker/NLS tanker ' 'Gas Carrier/NLS tanker']
Low risk types: ['Ro-Ro cargo' 'Container' 'General cargo/multipurpose' 'Tug'
 'Other special activities' 'Dredger' nan 'Offshore supply'
 'Refrigerated cargo' 'Heavy load' 'Warship and naval auxiliary'
 'MODU & FPSO' 'Fishing vessel' 'Pleasure yacht not engaged in trade'
 'Commercial yacht' 'Fish factory' 'Special purpose ship'
 'Government ship used for non-commercial purpose' 'High speed cargo'
 'Livestock carrier' 'Wooden ship of a primitive build']<jupyter_text>Flag<jupyter_code>cols = ['flag_in_pmou', 'flag_code']
df = portcalls_cleaned[cols].drop_duplicates().sort_values(cols)
flags_in_pmou = df.loc[lambda x: x['flag_in_pmou'], 'flag_code'].values
flags_not_in_pmou = df.loc[lambda x: ~x['flag_in_pmou'], 'flag_code'].values
print(f"Flag in pmou: {flags_in_pmou}")
print(f"Flag not in pmou: {flags_not_in_pmou}")
portcalls_cleaned['flag_in_pmou'].value_counts(normalize=True)
portcalls_cleaned['flag'].value_counts(normalize=True)
portcalls_cleaned['risk'].value_counts().sort_index()
display(
    portcalls_cleaned[['flag_in_pmou', 'risk']]
    .value_counts()
    .sort_index()
    .reset_index()
    .pivot(['flag_in_pmou'], ['risk'], 0)
    .apply(lambda x: x/ x.sum(), axis='columns')
)

display(
    portcalls_cleaned[['flag_in_pmou', 'risk']]
    .value_counts()
    .sort_index()
    .reset_index()
    .pivot(['flag_in_pmou'], ['risk'], 0)
)<jupyter_output><empty_output><jupyter_text>Country of port<jupyter_code>iso3166alpha2 = portcalls_processed['port'].str[:2].unique()
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
url = "https://gist.github.com/tadast/8827699/raw/f5cac3d42d16b78348610fc4ec301e9234f82821/countries_codes_and_coordinates.csv"
iso3166 = pd.read_csv(url).apply(lambda x: x.str.replace('"', "").str.strip())

mask = iso3166.set_index('Alpha-2 code')['Alpha-3 code'].to_dict()
iso3166alpha3 = [mask[country] for country in portcalls_cleaned['port'].str[:2].unique()]
fig, ax = plt.subplots(figsize=(15,10))
world.plot(ax=ax, color='red')
world.loc[lambda x: x.iso_a3.isin(iso3166alpha3)].plot(color='green', ax=ax)<jupyter_output><empty_output><jupyter_text>Sent At<jupyter_code>portcalls_cleaned['Sent.At'].agg(['min', 'max'])<jupyter_output><empty_output><jupyter_text>Stats<jupyter_code>first_inspection = inspections_processed.groupby('IMO')['DateOfFirstVisit'].first().to_dict()

result = dict()
ship_not_inspected = list()

for ship, group in tqdm(portcalls_processed.groupby('ship')):
    if ship in first_inspection:
        result[ship] = (group['arrival'] < first_inspection[ship]).sum()
    else:
        ship_not_inspected.append(ship)
result = pd.Series(result)

{
    'number of portcalls': len(portcalls_processed),
    'number of ships': portcalls_processed.ship.nunique(),
    'number of ships without any portcalls': len(set(inspections_processed['IMO'].unique()) - set(portcalls_processed['ship'].unique())),
    'number of ships not inspected': len(ship_not_inspected),
    'fraction of ships not inspected': len(ship_not_inspected) / portcalls_processed.ship.nunique(),
}<jupyter_output><empty_output><jupyter_text>Number of portcalls before inspection<jupyter_code>fig, ax = plt.subplots(figsize=(4.82,3.21), constrained_layout=True)
sns.ecdfplot(list(result.values), stat='proportion', ax=ax)
ax.set_xlim(0, 20)
ax.set_xticks(range(20))
ax.set_yticks(np.linspace(0, 1, 11))
ax.grid()
ax.set_xlabel('Number of portcalls before inspection')<jupyter_output><empty_output><jupyter_text>Some ships with many portcalls<jupyter_code>pd.Series(result).sort_values().tail(10)<jupyter_output><empty_output><jupyter_text>Ship flag history<jupyter_code>portcalls = (
    portcalls_cleaned
    .loc[lambda x: src.START_DATE <= x['arrival']]
    .loc[lambda x: x['arrival'] <= src.END_DATE]
    .loc[lambda x: x['ship'].duplicated(keep=False)]
    .dropna(subset=['flag'])
    .astype({'flag': 'int'})
)

result = list()
result2 = list()
number_of_changes_list = list()
for ship, group in portcalls.groupby('ship')['flag']:
    number_of_changes = (group.values != group.shift(1).values).sum() - 1
    number_of_changes_list.append(number_of_changes)
    if number_of_changes == 0:
        result.append(group.iloc[0])
    elif number_of_changes == 1:
        result2.append((group.iloc[0], group.iloc[-1]))
result = pd.Series(result)

total = len(number_of_changes_list)
print(f"Total number of ships: {total}")
print()
not_changing = (np.array(number_of_changes_list) == 0).sum()
changing = (np.array(number_of_changes_list) > 0).sum()
print(f"Total number of ships not changing flag: {not_changing} ({not_changing/total:.1%})")
print(f"Total number of ships changing flag: {changing} ({changing/total:.1%})")
print()
print(f"Total changes of flag: {np.bincount(number_of_changes_list)}")
np.set_printoptions(suppress=True)
message = 100*np.bincount(number_of_changes_list) / len(number_of_changes_list)
print(f"Total changes of flag (relative): {message}")
print()
message = "From the ships that don't change flag, number of white/grey/black ships"
print(f"{message}: {np.bincount(result)}")
print()
print("Type of changes:")
from collections import Counter
print(Counter(result2))
print()
more_worse_flag = np.bincount([a>b for a, b in result2]) / changing
print(f"Fraction of ships moving to better/worse flag: {more_worse_flag}")<jupyter_output>Total number of ships: 28416

Total number of ships not changing flag: 27647 (97.3%)
Total number of ships changing flag: 769 (2.7%)

Total changes of flag: [27647   722    44     2     1]
Total changes of flag (relative): [97.29377815  2.54082207  0.15484234  0.00703829  0.00351914]

From the ships that don't change flag, number of white/grey/black ships: [26300   672   675]

Type of changes:
Counter({(0, 2): 271, (1, 0): 195, (0, 1): 132, (2, 0): 85, (1, 2): 29, (2, 1): 10})

Fraction of ships moving to better/worse flag: [0.56176853 0.37711313]
{'fraction_non_alphanumeric': 0.15717884130982368, 'fraction_numerical': 0.0327455919395466, 'mean_word_length': 6.893638170974155, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 45, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '232215', 'license': 'MIT', 'n_tokens_mistral': 3096, 'n_tokens_neox': 2912, 'n_words': 571, 'path': '01-preprocessing-stats.ipynb', 'repo_name': 'gerritjandebruin/ship-risk-profile', 'size': '7940'}