<jupyter_start><jupyter_text><jupyter_code>super_srag = spark.read.parquet('gs://ai-covid19-datalake/standard/super-srag/super_srag_v2.parquet')
print('Super SRAG has {} records'.format(super_srag.count()))
super_srag.limit(5).toPandas()<jupyter_output>Super SRAG has 3726465 records<jupyter_text><jupyter_code>super_srag.select(['CLASSI_FIN', 'PP_PCR_RESUL', 'CRITERIO']).printSchema()<jupyter_output>root
 |-- CLASSI_FIN: integer (nullable = true)
 |-- PP_PCR_RESUL: string (nullable = true)
 |-- CRITERIO: string (nullable = true)<jupyter_text>Creating filtered data for not covid<jupyter_code>filter_not_covid = ((F.col('CLASSI_FIN') ==  1) | (F.col('CLASSI_FIN') ==  2))
super_srag.filter(filter_not_covid).write.parquet('gs://ai-covid19-datalake/trusted/super-srag_filters/super-srag_notcovid_nofilter.parquet', mode='overwrite')
print('Dataset with negative covid cases (without any filter): {}'.format(super_srag.filter(filter_not_covid).count()))<jupyter_output>Dataset with negative covid cases (without any filter): 44538<jupyter_text>Creating filtered data for positive covid<jupyter_code>filter_positive = (F.col('CLASSI_FIN') == 5)
filter_pcr_positive = ((F.col('PP_PCR_RESUL') ==  '2'))
filter_lab_positive = ((F.col('CRITERIO') ==  '1'))<jupyter_output><empty_output><jupyter_text>covid (without filter)<jupyter_code>super_srag.filter(filter_positive).write.parquet('gs://ai-covid19-datalake/trusted/super-srag_filters/super-srag_covid_nofilter.parquet', mode='overwrite')
print('Dataset with positive covid cases (without any filter): {}'.format(super_srag.filter(filter_positive).count()))<jupyter_output>Dataset with positive covid cases (without any filter): 2454142<jupyter_text>covid (positive pcr)<jupyter_code>super_srag.filter(filter_positive).filter(filter_pcr_positive).write.parquet('gs://ai-covid19-datalake/trusted/super-srag_filters/super-srag_covid_pcrpositive.parquet', mode='overwrite')
print('Dataset with positive covid cases (filtered by positive PCR): {}'.format(super_srag.filter(filter_positive).filter(filter_pcr_positive).count()))<jupyter_output>Dataset with positive covid cases (filtered by positive PCR): 1507881<jupyter_text>covid (positive lab)<jupyter_code>super_srag.filter(filter_positive).filter(filter_lab_positive).write.parquet('gs://ai-covid19-datalake/trusted/super-srag_filters/super-srag_covid_labpositive.parquet', mode='overwrite')
print('Dataset with positive covid cases (filtered by LAB criteria): {}'.format(super_srag.filter(filter_positive).filter(filter_lab_positive).count()))<jupyter_output>Dataset with positive covid cases (filtered by LAB criteria): 2157545<jupyter_text>Creating Super SRAG for reclassification<jupyter_code>filter_reclass = (F.col('CLASSI_FIN') ==  4)
super_srag.filter(filter_reclass).write.parquet('gs://ai-covid19-datalake/trusted/super-srag_filters/super-srag_toreclassify.parquet', mode='overwrite')
print('Dataset with positive covid cases (filtered by LAB criteria): {}'.format(super_srag.filter(filter_reclass).count()))
3*3*500<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14327772325809618, 'fraction_numerical': 0.018972849198560682, 'mean_word_length': 9.544827586206896, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '639315', 'license': 'MIT', 'n_tokens_mistral': 1143, 'n_tokens_neox': 1088, 'n_words': 149, 'path': '04-preprocessing-for-models/02-experiment-design/06-break-super-srag.ipynb', 'repo_name': 'pierrepita/super-srag-ai-covid19-google-cidacs', 'size': '3057'}