<jupyter_start><jupyter_text>SummaryPrepare the training / validation / test datasets using Apache Beam.----  Imports<jupyter_code>import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.runners.interactive.caching import CacheManager<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>options = PipelineOptions(
    project="deep-protein-gen", temp_location="gs://strokach-dataflow-temp", streaming=False,
)
options.display_data()<jupyter_output>WARNING:root:Discarding unparseable args: ['-f', '/home/kimlab1/strokach/.jupyter/data/runtime/kernel-9d5e2917-9685-4dd3-bd0f-7410ddc2cf42.json']<jupyter_text>Workflow<jupyter_code>p = beam.Pipeline()

p = (
    #
    p
    | beam.io.ReadAllFromParquet("gs://deep-protein-gen/training_data/database_id=G3DSA%253A1.10.10.180")
    | beam
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12669683257918551, 'fraction_numerical': 0.03733031674208145, 'mean_word_length': 6.7631578947368425, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '490904', 'license': 'MIT', 'n_tokens_mistral': 345, 'n_tokens_neox': 322, 'n_words': 36, 'path': 'notebooks/02_prepare_dataset_beam.ipynb', 'repo_name': 'ostrokach/proteinsolver', 'size': '884'}