<jupyter_start><jupyter_text>Parsing votes> Downloading & parsing votes Aafter downloading xlsx files behind the links on `https://www.bundestag.de/parlament/plenum/abstimmung/liste`.<jupyter_code>%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Setup<jupyter_code>from fastcore.all import *
from bundestag import html_parsing as hp
dry = True<jupyter_output><empty_output><jupyter_text>Collecting URIs for `.xlsx`/`.xls` documents from `.htm` files `.xlsx` / `.xls` will be referred as "sheet" files.<jupyter_code>path = Path('../bundestag.de_data')
html_path = path / 'htm_files'
sheet_path = path / 'sheets'
html_file_paths = hp.get_file_paths(html_path, pattern=hp.RE_HTM)
html_file_paths[:3]
hp.test_file_paths(html_file_paths, html_path)
%%time
sheet_uris = hp.collect_sheet_uris(html_file_paths)
list(sheet_uris.items())[:3], list(sheet_uris.items())[-3:]
hp.test_sheet_uris(sheet_uris)<jupyter_output><empty_output><jupyter_text>Downloading sheet files<jupyter_code>%%time
uri = sheet_uris['10.09.2020: Abstrakte Normenkontrolle - Düngeverordnung (Beschlussempfehlung)']
hp.download_sheet(uri, sheet_path=sheet_path, dry=dry)
%%time
hp.download_multiple_sheets(sheet_uris, 
                            sheet_path=sheet_path, 
                            nmax=3,
                            dry=dry)
%%time
file_title_maps = hp.get_file2poll_maps(sheet_uris, sheet_path)
hp.test_file_title_maps(file_title_maps, sheet_uris)<jupyter_output><empty_output><jupyter_text>Loading sheets into DataFrames Collecting the `xlsx` and `xls` file names<jupyter_code>sheet_files = hp.get_file_paths(sheet_path, pattern=hp.RE_FNAME)
sheet_files<jupyter_output><empty_output><jupyter_text>Reading files into dataframes<jupyter_code>%%time
sheet_file = sheet_files[0]
df = hp.get_sheet_df(sheet_file, 
                     file_title_maps=file_title_maps)
hp.test_get_sheet_df(df)
df.head().T<jupyter_output><empty_output><jupyter_text>Squishing vote columns<jupyter_code>df_squished = hp.get_squished_dataframe(df)
hp.test_squished_df(df_squished, df)
df_squished.head(3).T<jupyter_output><empty_output><jupyter_text>Setting some dtypes<jupyter_code>df_squished = hp.set_sheet_dtypes(df_squished)<jupyter_output><empty_output><jupyter_text>Loading multiple sheets into dataframes<jupyter_code>%%time
df = hp.get_multiple_sheets_df(sheet_files, file_title_maps=file_title_maps, dry=dry)
df.head().T<jupyter_output><empty_output><jupyter_text>Doing all the above<jupyter_code>%%time
df = hp.get_multiple_sheets(html_path, sheet_path, nmax=3, dry=dry)
df.head().T<jupyter_output><empty_output><jupyter_text>Writing to disk<jupyter_code>%%time
# df.to_parquet('../bundestag.de_votes.parquet')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1153705732018985, 'fraction_numerical': 0.006206644760861628, 'mean_word_length': 7.353658536585366, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 46, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '647481', 'license': 'Apache-2.0', 'n_tokens_mistral': 1097, 'n_tokens_neox': 1060, 'n_words': 136, 'path': 'nbs/00_html_parsing.ipynb', 'repo_name': 'eschmidt42/bundestag', 'size': '2739'}