<jupyter_start><jupyter_text>Summary---  Imports<jupyter_code>%run _imports.ipynb
import pmc_tables
importlib.reload(pmc_tables)
NOTEBOOK_NAME = 'load_pmc_xml_tables'
os.makedirs(NOTEBOOK_NAME, exist_ok=True)<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>MANUSCRIPT_DATA_DIR = Path('../downloads/ncbi/pmc/manuscript/').absolute()<jupyter_output><empty_output><jupyter_text>Load test data  Manuscripts<jupyter_code>!ls {MANUSCRIPT_DATA_DIR}
manuscript_files = pd.read_csv('../downloads/ncbi/pmc/manuscript/filelist.csv')
manuscript_files.head()<jupyter_output><empty_output><jupyter_text>Process  Test 1<jupyter_code>manuscript_file = MANUSCRIPT_DATA_DIR.joinpath('PMC0021XXXXX/PMC2136438.xml')
manuscript_data = pmc_tables.extract_tables_from_xml(manuscript_file)
data.keys()
data['/PMC2136438.xml/T7-0']
table_bytes = pmc_tables.decompress_from_b85(data['/PMC2136438.xml/T7-0']['table'])
display(HTML(table_bytes.decode('ascii')))
import pmc_tables
# with pd.HDFStore(f'{NOTEBOOK_NAME}/pmc-tables-1.h5', 'w') as store:

#     for i, row in enumerate(manuscript_files.itertuples()):
#         print('\n' + f"{row.File}")

#         results = {}
#         xml_file = MANUSCRIPT_DATA_DIR.joinpath(row.File)

#         count += 1
#         if count > 1_000:
#             break<jupyter_output><empty_output><jupyter_text>Example `xml-wrap`<jupyter_code>tree = ET.parse(MANUSCRIPT_DATA_DIR.joinpath('PMC0021XXXXX/PMC2136438.xml'))
root = tree.getroot()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14142761841227486, 'fraction_numerical': 0.0333555703802535, 'mean_word_length': 6.731958762886598, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '644551', 'license': 'MIT', 'n_tokens_mistral': 651, 'n_tokens_neox': 594, 'n_words': 55, 'path': 'notebooks/xx-load_pmc_xml_tables.ipynb', 'repo_name': 'pmc-tables/pmc-tables', 'size': '1499'}