<jupyter_start><jupyter_text>Dataset Statistics  Dataset Creation<jupyter_code># TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
_CITATION = """\
@InProceedings{huggingface:dataset,
title = {HIV Full Geneome Dataset},
author={<NAME>
},
year={2021}
}
"""

# TODO: Add a link to an official homepage for the dataset here
_HOMEPAGE = ""

# TODO: Add the licence for the dataset here if you can find it
_LICENSE = ""

# TODO: Add description of the dataset here
# You can copy an official description
_DESCRIPTION = """\
This dataset was constructured from the LANL Full Genome Database v2016.
Genes were extracted from the genome using the GeneCutter webtool provided by the database.
"""

genome_info = datasets.DatasetInfo(description = _DESCRIPTION,
                                  homepage=_HOMEPAGE, license = _LICENSE, citation=_CITATION)
genome_dset = datasets.Dataset.from_pandas(info_df,
                                          info = genome_info)
genome_dset
genome_dset.save_to_disk(dataset_root + 'FLT_genome')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07880434782608696, 'fraction_numerical': 0.007246376811594203, 'mean_word_length': 3.9551569506726456, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '75375', 'license': 'CECILL-B', 'n_tokens_mistral': 342, 'n_tokens_neox': 321, 'n_words': 109, 'path': 'workflow/scripts/process_lanl_flt.py.ipynb', 'repo_name': 'DamLabResources/hiv-transformers', 'size': '1110'}