<jupyter_start><jupyter_text>```         Copyright <NAME> 2020-2021.Distributed under the Boost Software License, Version 1.0.   (See accompanying file LICENSE_1_0.txt or copy at         http://www.boost.org/LICENSE_1_0.txt)```  Re-implementing <NAME>'s [Code Review Stack Exchange post](https://codereview.stackexchange.com/questions/239521/performance-read-large-amount-of-xmls-and-load-into-single-csv)<jupyter_code>import os

import pandas as pd
from tqdm import tqdm

import scrape.nct as nct
# https://clinicaltrials.gov/ct2/resources/download#DownloadAllData
nct_url = 'https://clinicaltrials.gov/AllPublicXML.zip'
file = nct_url.split('/')[-1]
path = file.split('.')[0]
nct.download_data(nct_url, file)    # 1.6 Gb on disk
nct.unzip_data(file, path)  # 8.5 Gb on disk
xml_files = [
    os.path.join(dirpath, file)
    for dirpath, _, filenames in os.walk(path)
    for file in filenames
    if file.endswith('.xml')
]
df = pd.concat((
    nct.parse_xml(file)
    for file in tqdm(xml_files)
))
df.info()   # 3.0 Gb in memory, 12.0 Gb on disk
df.head()
df.tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13830755232029118, 'fraction_numerical': 0.029117379435850774, 'mean_word_length': 4.092592592592593, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '112488', 'license': 'BSL-1.0', 'n_tokens_mistral': 439, 'n_tokens_neox': 410, 'n_words': 90, 'path': 'examples/nct.ipynb', 'repo_name': 'rhalbersma/scrape', 'size': '1115'}