<jupyter_start><jupyter_text>Output a set of validated transitionsHere, we output a curated table with transitions for all top level ESCO occupations, that are both `safe and desirable` according to the career transitions algorithm, and `feasible` according to the crowdsourced feasibility judgments (or predicted feasibility judgements if crowd data was not available).  0. Import dependencies and inputs<jupyter_code>%run ../notebook_preamble_Transitions.ipy
import mapping_career_causeways.models.predict_model as model
import os 

data = load_data.Data()
sims = load_data.Similarities()
trans_utils.find_most_similar(139)
# Input data path
safe_transition_filepath = f'{data_folder}processed/validation/transitions_Safe_Desirable_Validated.csv'

# Output data
output_folder = useful_paths.project_dir + '/supplementary_online_data/transitions/transitions_tables/'
file_name = 'mcc_transitions_table_Validated.csv'<jupyter_output><empty_output><jupyter_text>1. Get feasibility estimates for all safe and desirable transitions<jupyter_code># Find all transitions above the minimal similarity threshold
transitions_master_table = trans_utils.get_transitions(verbose=True)
# Add crowdsourced and predicted feasibility ratings
if os.path.exists(safe_transition_filepath):
    safe_transitions = pd.read_csv(safe_transition_filepath)
else:
    # (will take some while for generating all the predictions)
    safe_transitions = transitions_master_table[transitions_master_table.is_safe_desirable]
    safe_transitions = model.add_feasibility_predictions(safe_transitions, add_crowd_data=True)
    safe_transitions.to_csv(safe_transition_filepath, index=False)

safe_transitions.head(1)
# Select only transitions that are also predicted to be feasible by the predictive model or the crowd
safe_validated_transitions = safe_transitions[
    safe_transitions.is_feasible_predicted | (safe_transitions.is_feasible)
]
print(f'Total number of safe and desirable transitions: {len(safe_transitions)}')
print(f"Total number of 'feasible' safe and desirable transitions: {len(safe_validated_transitions)}")
safe_validated_transitions.info()
sns.distplot(safe_validated_transitions.groupby('origin_id').count().destination_id)
plt.xlabel('number of transitions per occupation')
plt.show()
# Fraction of all top level ESCO occupations featured in the table as origin/destination 
print(len(safe_validated_transitions.origin_id.unique()) / len(data.report_occ_ids))
print(len(safe_validated_transitions.destination_id.unique()) / len(data.report_occ_ids))<jupyter_output>0.9293177627535341
0.828518746158574<jupyter_text>2. Manual curation<jupyter_code># It might be confusing to be recommended transitions to these roles;
# hence, we take them out
filter_out = [
    'escort',
    'fortune teller', 'psychic', 'medium',
]
curated_transitions = safe_validated_transitions.copy()
curated_transitions = curated_transitions[-curated_transitions.origin_label.isin(filter_out)]
curated_transitions = curated_transitions[-curated_transitions.destination_label.isin(filter_out)]
print(f"Total number of 'feasible', curated safe and desirable transitions: {len(curated_transitions)}")<jupyter_output>Total number of 'feasible', curated safe and desirable transitions: 26642<jupyter_text>3. Output a curated table in supplementary online data<jupyter_code># Add risk category
transitions = curated_transitions.copy()
transitions=data.add_field_to_occupation(transitions, 'origin_id', 'risk_category', 'origin')
transitions=data.add_field_to_occupation(transitions, 'destination_id', 'risk_category', 'destination')

for only_high_risk_cols in ['is_safer', 'is_strictly_safe', 'is_strictly_safe_desirable']:
    transitions.loc[-(transitions.origin_risk_category=='High risk'), only_high_risk_cols] = np.nan
# Reorganise the dataframe
transitions = transitions[[
    'origin_id', 'origin_label', 'destination_id', 'destination_label',
    'similarity', 'sim_category', 'feasibility', 'is_feasible', 'feasibility_predicted', 'is_feasible_predicted',
    'W_skills', 'W_work',
    'W_essential_skills', 'W_optional_skills', 'W_activities', 'W_work_context',
    'job_zone_dif', 'earnings_ratio', 'risk_dif', 'prop_dif',
    'origin_risk_category', 'destination_risk_category',
    'is_jobzone_ok', 'is_earnings_ok', 'is_not_high_risk', 'is_safer', 'is_strictly_safe',
    'is_viable', 'is_desirable', 'is_safe_desirable', 'is_strictly_safe_desirable'
]]
transitions.sample(3)
transitions.info()
transitions.to_csv(output_folder + file_name, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09306409130816505, 'fraction_numerical': 0.010316066725197541, 'mean_word_length': 8.35728952772074, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '476677', 'license': 'MIT', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1404, 'n_words': 310, 'path': 'codebase/notebooks/07_validation/Validation_04_Output_validated_transitions.ipynb', 'repo_name': 'maxdata/mapping-career-causeways', 'size': '4556'}