<jupyter_start><jupyter_text><jupyter_code>results = spark.read.csv('gs://ai-covid19-datalake/trusted/results_google_ai_seminario.csv', header=True)
results.limit(10).toPandas()
float_cols = ['model_AUC_ROC', 'model_AUC_PR', 'model_covid_precision', 'model_covid_recall', 'model_covid_f1', 'model_not_covid_precision', 'model_not_covid_recall', 'model_not_covid_f1', 'model_avg_precision', 'model_avg_recall', 'model_avg_f1', 'model_avg_acc']
for col in float_cols:
    results = results.withColumn(col, F.col(col).cast('float'))
results.groupBy(['model_name', 'experiment_filter','undersampling_method', 'undersampling_column', 'model_col_set']).count()\
       .orderBy(['model_name', 'experiment_filter','undersampling_method', 'undersampling_column', 'model_col_set']).show(100)
cols = ['model_name', 'experiment_filter','undersampling_method', 'undersampling_column', 'model_col_set']

results.groupBy(cols)\
        .agg({'model_avg_precision': 'mean',
              'model_avg_recall': 'mean',
              'model_avg_f1': 'mean'})\
        .orderBy(cols).toPandas()
cols = ['model_name', 'experiment_filter','undersampling_method', 'undersampling_column', 'model_col_set']

results.groupBy(cols)\
       .agg({'model_avg_precision': 'stddev',
              'model_avg_recall': 'stddev',
              'model_avg_f1': 'stddev'})\
        .orderBy(cols).toPandas()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16416845110635261, 'fraction_numerical': 0.008565310492505354, 'mean_word_length': 6.22680412371134, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '553150', 'license': 'MIT', 'n_tokens_mistral': 522, 'n_tokens_neox': 501, 'n_words': 58, 'path': '06-results/11-find_best_model_from_table.ipynb', 'repo_name': 'pierrepita/super-srag-ai-covid19-google-cidacs', 'size': '1401'}