<jupyter_start><jupyter_text>iMeta classification algorithm demoThe work of the XBT project builds on previous work to create an algorithm to fill in missing metadata relating to probe type and manufacturer in the XBt dataset. This algorithm is called the iMeta algorithm and is decribed in this paper:https://journals.ametsoc.org/doi/full/10.1175/JTECH-D-17-0129.1It is a deterministic algorithm which uses the country, maximum depth of profile and date of profile as input. The frequency of probe types by date was studied and the assignment of probe model and manufacturer using the three input values is based on the changes in the frequency of observed types over time to determine the most likely model and manufacturer.This flow chart describes the algorithm, which has been implemented in the code used by this notebook.<jupyter_code>import os
import sys
from pathlib import Path

import pandas
import numpy
import sklearn
import sklearn.metrics
import matplotlib
import matplotlib.pyplot
root_repo_dir = Path().absolute().parent
sys.path = [str(root_repo_dir)] + sys.path
from xbt.dataset import XbtDataset
from xbt.imeta import imeta_classification, XBT_MAX_DEPTH
# TODO: access through a intake catalog
# Set up some site specific parameters for the notebook
try:
    environment = os.environ['XBT_ENV_NAME']
except KeyError:
    environment = 'pangeo'
env_data_dirs = {
    'MO_scitools': '/data/users/shaddad/xbt-data/csv_with_imeta',
    'pangeo': '/data/misc/xbt-data/csv_with_imeta/',
}
env_date_ranges = {
    'MO_scitools': (1966,2015),
    'pangeo': (1966,2015)
}
result_dirs = {
    'MO_scitools': '/data/users/shaddad/xbt-data/results',
    'pangeo': '/data/misc/xbt-data/results/',   
}

# Set up some dataset specific parameters
xbt_data_dir = env_data_dirs[environment]
DATA_YEAR_RANGE=env_date_ranges[environment]
columns = ['year','num obs ''num cruises', 'unknown brand or type',
           'unknown type', 'num_instruments','num_platforms']
result_dir = result_dirs[environment]
xbt_full_dataset = XbtDataset(xbt_data_dir, env_date_ranges[environment])
xbt_labelled = xbt_full_dataset.filter_obs({'labelled': 'labelled'})

imeta_classes = xbt_labelled.xbt_df.apply(imeta_classification, axis=1)
imeta_model = imeta_classes.apply(lambda t1: t1[0])
imeta_manufacturer = imeta_classes.apply(lambda t1: t1[1])

imeta_instrument = imeta_classes.apply(lambda t1: f'XBT: {t1[0]} ({t1[1]})') 
actual_classes = xbt_labelled.xbt_df.apply(lambda row1: (row1['model'],row1['manufacturer']), axis=1)<jupyter_output><empty_output><jupyter_text>Maximum depth of profiles for different probe typesThese plots show the max depth of profiles for different probe types, which is used by the iMeta algorithm.<jupyter_code>instruments_to_plot = [
   'XBT: T4 (SIPPICAN)',
   'XBT: T5 (SIPPICAN)',
   'XBT: T7 (SIPPICAN)',
   'XBT: T10 (SIPPICAN)',
   'XBT: DEEP BLUE (SIPPICAN)',
   'XBT: FAST DEEP (SIPPICAN)',
   'XBT: T4 (TSK - TSURUMI SEIKI Co.)',
   'XBT: T5 (TSK - TSURUMI SEIKI Co.)',
   'XBT: T6 (TSK - TSURUMI SEIKI Co.)',    
   'XBT: T7 (TSK - TSURUMI SEIKI Co.)',
   'XBT: T10 (TSK - TSURUMI SEIKI Co.)',
]
depth_bins = list(map(float, list(range(0,2001,50))))
fig_probe_freq = matplotlib.pyplot.figure('probe frequency plot grid',figsize=(24,18))
for ix1, mfi in enumerate(instruments_to_plot):
    ax_depth_dist_type = fig_probe_freq.add_subplot(4,3,ix1+1,title=f'max depth distribution {mfi}')
    _ = xbt_labelled.filter_obs({'instrument': mfi})['max_depth'].hist(bins=depth_bins,ax=ax_depth_dist_type)<jupyter_output><empty_output><jupyter_text>Frequency of probe type by yearThese plots show the frequency of different probe types by year of observation. This data was used to determine date thresholds in the iMeta algorithm.<jupyter_code>plot_pairs = [ 
    ('XBT: T10 (SIPPICAN)', 'XBT: T4 (SIPPICAN)', lambda d1: d1<=360.0),
    ('XBT: T5 (SIPPICAN)', 'XBT: FAST DEEP (SIPPICAN)', lambda d1: d1>= 1000.0 and d1 <=1350.0),
    ('XBT: T7 (SIPPICAN)', 'XBT: DEEP BLUE (SIPPICAN)',lambda d1: d1>= 600.0 and d1 <=1000.0),
]
fig_freq_comp = matplotlib.pyplot.figure('plot types for year and max depth',figsize=(18,6))
l1 = []
for ix1, (type1, type2, filter1) in enumerate(plot_pairs):
    dataset_type1 = xbt_labelled.filter_obs({'instrument':type1}).xbt_df
    profiles_type1 = dataset_type1[dataset_type1.max_depth.apply(filter1)]
    dataset_type2 = xbt_labelled.filter_obs({'instrument':type2}).xbt_df
    profiles_type2 = dataset_type2[dataset_type2.max_depth.apply(filter1)]
    type_year_dist1 = profiles_type1.year.value_counts(sort=False)
    type_year_dist2 = profiles_type2.year.value_counts(sort=False)
    freq_df1 = pandas.DataFrame({'year': list(type_year_dist1.index), 'num_profiles': list(type_year_dist1.values)}).sort_values('year')
    freq_df2 = pandas.DataFrame({'year': list(type_year_dist2.index), 'num_profiles': list(type_year_dist2.values)}).sort_values('year')
    l1 += [(freq_df1, freq_df2)]
    ax1 = fig_freq_comp.add_subplot(130 + ix1 + 1)
    ax1.plot(list(freq_df1.year), list(freq_df1.num_profiles), color='b')
    ax1.plot(list(freq_df2.year), list(freq_df2.num_profiles), color='g')
    ax1.xmin=1950
    ax1.xmax=2020<jupyter_output><empty_output><jupyter_text>Number of correctly classified profilesThese plots show the number of profiles per year compared to the number of profiles with metadata, and what proportion of those profiles are correctly labelled (in terms of model and manufacturer) by the iMeta algorithm.<jupyter_code>imeta_correct = [sum(actual_classes[xbt_labelled['year'] == year1] == imeta_classes[xbt_labelled['year'] == year1]) for year1 in range(DATA_YEAR_RANGE[0], DATA_YEAR_RANGE[1]+1)]
probe_type_correct = [sum(xbt_labelled['model'][xbt_labelled['year'] == year1] == imeta_model[xbt_labelled['year'] == year1]) for year1 in range(DATA_YEAR_RANGE[0], DATA_YEAR_RANGE[1]+1)]
df_probe_labels = pandas.DataFrame.from_dict({
    'year': xbt_full_dataset['year'].value_counts(sort=False).index, 
    'total_profiles': xbt_full_dataset['year'].value_counts(sort=False).values,
    'labelled_profiles': xbt_labelled['year'].value_counts(sort=False).values,
    'imeta_correct': imeta_correct,
    'probe_type_correct': probe_type_correct, 
})
fig_probe_labels = matplotlib.pyplot.figure('correct_probe_labels')
ax_probe_labels = fig_probe_labels.add_subplot(111)
_ = df_probe_labels.plot(ax=ax_probe_labels, 
                     x='year',
                     y=['total_profiles','labelled_profiles','imeta_correct','probe_type_correct'],
                     figsize=(18,12),
                     title='Number of correctly labelled probes',
                     color=['k','b','orange','r'])
fig_probe_labels.savefig(os.path.join(result_dir, 'correct_probe_labels.png'))<jupyter_output><empty_output><jupyter_text>Accuracy of iMeta algorithmThe following plots analyse the accuracy of the iMeta algorithm. The metric used in the iMeta paper is recall. This is a measure of how many data points belonging to class A have been correctly classified as class A. The definition is (taken from [wikipedia](https://en.wikipedia.org/wiki/Precision_and_recall)```recall = (number of data points classified as class A that belong to class A) / (number of datapoints that belong to class A)```<jupyter_code>imeta_instrument = imeta_classes.apply(lambda t1: 'XBT: {0} ({1})'.format(*t1))
le1 = sklearn.preprocessing.LabelEncoder()
le1.fit(xbt_labelled['instrument'])

actual_encoded = le1.transform(xbt_labelled['instrument'])
imeta_encoded = le1.transform(imeta_instrument)

(precision, recall, fscore, support
    ) = sklearn.metrics.precision_recall_fscore_support(actual_encoded, imeta_encoded)
metrics_df = pandas.DataFrame({'instrument': le1.classes_,
                               'precision': precision,
                               'recall': recall,
                               'f-score': fscore,
                               'support': support,
                              })

(precision_avg, recall_avg, fscore_avg, support_avg
    ) = sklearn.metrics.precision_recall_fscore_support(actual_encoded, imeta_encoded,average='micro')
print(f'average metrics for intelligent metadata algorithm on the labelled profiles: \n'
      f' precision: {precision_avg:.03f}\n'
      f' recall: {recall_avg:.03f}\n'
      f' f1 score: {fscore_avg:.03f}\n'
     )
fig_imeta_scores = matplotlib.pyplot.figure('imeta scores', figsize=(24,12))
ax_precision = fig_imeta_scores.add_subplot(121)
metrics_df.precision.plot.bar(ax=ax_precision, x='instrument', y='precision')
ax_recall = fig_imeta_scores.add_subplot(122)
metrics_df.precision.plot.bar(ax=ax_recall, x='instrument', y='recall')
xbt_full_dataset.xbt_df[:20].plot.bar(x='instrument',y='max_depth')<jupyter_output><empty_output><jupyter_text>Probe model frequenciesThese plots compare the distribution of probe model types for the profiles with model data compared with the assignments by the iMeta algorithm.<jupyter_code>actual_labels = list(actual_classes.value_counts().keys())
actual_counts = actual_classes.value_counts().values
if len(actual_labels) > 10:
    actual_labels = actual_labels[:10] + ['other']
    actual_counts = list(actual_counts[:10]) + [sum(actual_counts[10:])]
imeta_labels = list(imeta_classes.value_counts().keys())
imeta_counts = list(imeta_classes.value_counts().values)
fig_name_probe_freqs = 'xbt probe type frequencies'
current_figure = matplotlib.pyplot.figure(fig_name_probe_freqs, figsize=(24,12))
axes_actual_types = current_figure.add_subplot(121)
actual_plot_objs = axes_actual_types.pie(actual_counts,labels=actual_labels)
axes_imeta_types = current_figure.add_subplot(122)
imeta_plot_objs = axes_imeta_types.pie(imeta_counts,labels=imeta_labels)
current_figure.savefig(os.path.join(result_dir, fig_name_probe_freqs + '.png'))
fig_instr_freq_pie = matplotlib.pyplot.figure('instrument_freqs', figsize=(16,16))
ax_instr_freq_pie = fig_instr_freq_pie.add_subplot(1,1,1, title='Frequncies of different probe types (for known types)')
_ = pandas.DataFrame({'instrument': xbt_labelled.xbt_df.instrument.value_counts().index[:10],
                  'instrument frequency': xbt_labelled.xbt_df.instrument.value_counts().values[:10],
                 },index=xbt_labelled.xbt_df.instrument.value_counts().index[:10]).plot.pie(ax=ax_instr_freq_pie, y='instrument frequency', figsize=(16,16))<jupyter_output><empty_output><jupyter_text>Probe types for different subsets of countriesnext we want to see if there is going to be much predictability based on the country of origin, a feature used by imeta. We will look at how the instrument type frequewncies vary between the two sets of countries used in iMeta. The two groups are* East Asian countries: China, Taiwan, Republic of Korea, Japan* rest of the world<jupyter_code>xbt_east_asia= xbt_labelled.xbt_df[(xbt_labelled.xbt_df.country =='JAPAN') | (xbt_labelled.xbt_df.country =='KOREA; REPUBLIC OF') | (xbt_labelled.xbt_df.country =='CHINA') | (xbt_labelled.xbt_df.country =='TAIWAN')]
xbt_rest_of_world = xbt_labelled.xbt_df[(xbt_labelled.xbt_df.country !='JAPAN') & (xbt_labelled.xbt_df.country !='KOREA; REPUBLIC OF') & (xbt_labelled.xbt_df.country !='CHINA') & (xbt_labelled.xbt_df.country !='TAIWAN')]
figure_type_freqs_by_country = matplotlib.pyplot.figure('xbt instrument types by country', figsize=(24,12))
ax_type_freqs_east_asia = figure_type_freqs_by_country.add_subplot(1,2,1,title='instrument frequencies for East Asian countries')
_ = pandas.DataFrame({'instrument': xbt_east_asia.instrument.value_counts().index[:10],
                  'instrument frequency': xbt_east_asia.instrument.value_counts().values[:10],
                 },index=xbt_east_asia.instrument.value_counts().index[:10]).plot.pie(ax=ax_type_freqs_east_asia, y='instrument frequency')
ax_type_freqs_rest_of_world = figure_type_freqs_by_country.add_subplot(1,2,2,title='instrument frequencies for Rest of the world')
_ = pandas.DataFrame({'instrument': xbt_rest_of_world.instrument.value_counts().index[:10],
                  'instrument frequency': xbt_rest_of_world.instrument.value_counts().values[:10],
                 },index=xbt_rest_of_world.instrument.value_counts().index[:10]).plot.pie(ax=ax_type_freqs_rest_of_world, y='instrument frequency')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11999348746336698, 'fraction_numerical': 0.021735591012699448, 'mean_word_length': 5.604838709677419, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '280664', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4390, 'n_tokens_neox': 4163, 'n_words': 886, 'path': 'notebooks/imeta_demo.ipynb', 'repo_name': 'uk-gov-mirror/MetOffice.XBTs_classification', 'size': '12284'}