<jupyter_start><jupyter_text>Hansard Parsing Prereq: Dataset: hansard-xmlSee `GetHansard.ipynb` to get the list of urls for Austalian Hansard for a particular year and saved to the `data/remote/hansard-urls` - then use the following to be able to get the data.```data/get-dataset.sh hansard-xml | sed 's/YEAR=2020/YEAR=2019/g' | sh````<jupyter_code>import glob, io, re, json, multiprocessing
from collections import OrderedDict
import xmlplain
from bs4 import BeautifulSoup
from IPython.display import JSON
from tqdm import tqdm
import pkg_resources
from symspellpy import SymSpell, Verbosity
import nltk
nltk.download('punkt')
input_xml_files = glob.glob("{}/*/*/*".format(data_path))<jupyter_output><empty_output><jupyter_text>Removing xmlns attributes and other attributeshttps://stackoverflow.com/questions/9044088/beautifulsoup-strip-specified-attributes-but-preserve-the-tag-and-its-contents<jupyter_code>REMOVE_ATTRIBUTES = [
    'lang','language','onmouseover','onmouseout','script','style','font',
    'dir','face','size','color','style','class','width','height','hspace',
    'border','valign','align','background','bgcolor','text','link','vlink',
    'alink','cellpadding','cellspacing']

REMOVE_ATTRIBUTES += ['xmlns:WX', 'xmlns:a', 'xmlns:aml', 'xmlns:o', 'xmlns:pic', 
                      'xmlns:r', 'xmlns:v', 'xmlns:v', 'xmlns:wp', 'xmlns:w', 'xmlns:w10',
                     'role']

def remove_attributes_from_soup(soup):
    for attribute in REMOVE_ATTRIBUTES:
        for tag in soup.find_all(attrs={attribute: True}):
            del tag[attribute]
    return soup<jupyter_output><empty_output><jupyter_text>Removing tags but keep the texthttps://stackoverflow.com/questions/1765848/remove-a-tag-using-beautifulsoup-but-keep-its-contents<jupyter_code>REMOVE_TAGS = ['span', 'a']

def strip_tags_keep_text_from_soup(soup):
    for tag in REMOVE_TAGS: 
        for match in soup.find_all(tag):
            match.unwrap()
    return soup<jupyter_output><empty_output><jupyter_text>Combine consecutive p tags into onehttps://stackoverflow.com/questions/50026264/beautifulsoup-combine-consecutive-tags**Does not seem to be working**<jupyter_code>for p in soup.find_all('p'):
    try:
        if p.find_previous().name=='p':
            p.find_previous().text += p.text
            p.decompose()
    except:
        pass<jupyter_output><empty_output><jupyter_text>Tokenisation and Spellcheck<jupyter_code>sym_spell = SymSpell(max_dictionary_edit_distance=2, prefix_length=7)
dictionary_path = pkg_resources.resource_filename(
    "symspellpy", "frequency_dictionary_en_82_765.txt")
bigram_path = pkg_resources.resource_filename(
    "symspellpy", "frequency_bigramdictionary_en_243_342.txt")
sym_spell.load_dictionary(dictionary_path, term_index=0, count_index=1)
sym_spell.load_bigram_dictionary(bigram_path, term_index=0, count_index=2)

def spellcheck(sentence):
    suggestions = sym_spell.lookup_compound(sentence, max_edit_distance=2)
    return suggestions[0].term

def token_and_spellcheck(content, sentence_len_thresh = 50):
    sentences = nltk.tokenize.sent_tokenize(content)
    sentences = [spellcheck(x) for x in sentences if len(x) > sentence_len_thresh]
    return sentences<jupyter_output><empty_output><jupyter_text>Just make some ramblings<jupyter_code>def clean_body(text):
    body = re.split('<body>|</body>', text)
    #print(body)
    if len(list(body)) == 1:
        body = body[0]
    if len(list(body)) > 1:
        body = body[1]
    else:
        body = ''
    body = body.replace("<p>", '').replace('</p>', '')
    return body

def xml_to_ramblings(file_name):
    soup = BeautifulSoup(open(file_name), "lxml-xml")
    try:
        date = soup.date.get_text()
    except:
        date = file_name.split("/")[-1].split(".")[:-1]
    soup = remove_attributes_from_soup(soup)
    soup = strip_tags_keep_text_from_soup(soup)
    split_by_name = re.split('<name>|</name>',str(soup))[1:]
    name_text = list(zip(split_by_name[0::2], split_by_name[1::2]))
    ramblings = []
    topics = []
    for name,text in name_text:
        #print(name, text)
        t = re.findall('<title>(.+)<\/title>', text)
        if len(t) > 0:
            topics = t
        party = re.findall('<party>(.+)<\/party>', text)
        d = {'date' : date, 'name' : name, 'text' : clean_body(text), 'party' : party[0] if party else '', 'topics' : topics}
        
        if len(d['text'].split(" ")) > 50:
            sentences = token_and_spellcheck(d["text"])
            d['sentences'] = sentences
            ramblings.append(d)
    return ramblings<jupyter_output><empty_output><jupyter_text>Visualise<jupyter_code>soup = BeautifulSoup(open(input_xml_files[0]), "lxml-xml")
visualise = str(soup)
JSON(xmlplain.xml_to_obj(io.StringIO(visualise), strip_space=True))<jupyter_output><empty_output><jupyter_text>Loop and write out<jupyter_code>def process_and_write(i):
    file_name = i.split("/")[-1].split(".")[0:-1][0]
    year = i.split("/")[-2]
    house = i.split("/")[-3]
    #print("Processing: {}".format(file_name))
    #soup = BeautifulSoup(open(i), "lxml-xml")
    #remove_attributes_from_soup(soup)
    #strip_tags_keep_text_from_soup(soup)
    ramblings = xml_to_ramblings(i)
    if len(ramblings) < 1:
        return
    
    # Make output dir
    if not os.path.exists("{}/{}/{}".format(data_output_path, house, year)):
        os.makedirs("{}/{}/{}".format(data_output_path, house, year))
    with open("{}/{}/{}/{}.json".format(data_output_path, house, year, file_name), "w", encoding='utf-8') as file:
        json.dump(ramblings, file)
    
pool = multiprocessing.Pool()
output = list(tqdm(pool.imap(process_and_write, input_xml_files), total=len(input_xml_files)))
xml_to_ramblings(input_xml_files[333])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15272727272727274, 'fraction_numerical': 0.014025974025974027, 'mean_word_length': 4.543186180422265, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 47, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 31}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '260536', 'license': 'MIT', 'n_tokens_mistral': 2105, 'n_tokens_neox': 2061, 'n_words': 340, 'path': 'nb/ParseHansard.ipynb', 'repo_name': 'thaum-io/language-of-leadership', 'size': '5775'}