<jupyter_start><jupyter_text>Process html files<jupyter_code>order = {
    'h0': 0,
    'h1': 1,
    'h2': 2,
    'h3': 3,
    'h4': 4,
    'h5': 5,
    'h6': 6,
    'strong': 10,
    'em': 11,
    'p': 20,
    'a': 20,
    'div': 20,
    'span': 20,
    'ol': 21,
    'li': 21,
}

for document in documents:
    if document['format'] != 'html' or 'body' not in document:
        continue
    body = BeautifulSoup(document['body'], 'html.parser')
    comments = body.findAll(text=lambda text:isinstance(text, Comment))
    for comment in comments:
        comment.extract()

    content = []
    stack = [body]
    elements = body.find_all(text = True)
    for element in elements:
        if str(element) == 'here' or len(str(element).split()) == 0:
            continue
        temp = element
        size = 100
        while temp.parent:
            temp = temp.parent
            if temp.name == 'table' or temp.href:
                size = 100
                break
            if temp.name in order:
                size = min(size, order[temp.name])
        if size == 100:
            continue
        content.append((str(element), size))
        print(size, str(element))
    document['content'] = content
helper.write_json(documents, '../data/files/iiit_website_content.json')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1466565349544073, 'fraction_numerical': 0.030395136778115502, 'mean_word_length': 2.0771028037383177, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '485021', 'license': 'Apache-2.0', 'n_tokens_mistral': 467, 'n_tokens_neox': 428, 'n_words': 110, 'path': 'iiitd_policy_chatbot/notebooks/1.scrape_iiitd_website.ipynb', 'repo_name': 'kracr/iiitd-chatbot', 'size': '1316'}