<jupyter_start><jupyter_text>Preparing the HTML for articles 1 to 8 files before turning them into ePubs<jupyter_code>from bs4 import BeautifulSoup
import re
import os

for i in range(1, 9):
    try:
        if not os.path.exists(str(i)):
            os.makedirs(str(i))
            os.makedirs(str(i)+'/images')
    except OSError:
        print ('Error: Creating directory. ' +  directory)
    with open('isaw-papers/isaw-papers-'+str(i)+'/isaw-papers-'+str(i)+'.xhtml') as paper:
        soup = BeautifulSoup(paper)

    #suppress metatags
    selects = soup.findAll(['meta'])
    for match in selects:
        match.decompose()

    #suppress links, except for the one linking to the css fil
    links = soup.link.find_all()

    dcterms = soup.find_all("link", {"rel" : re.compile("dcterms*")})
    for match in dcterms :
        match.decompose()

    schema = soup.find_all("link", {"rel" : re.compile("schema*")})
    for match in schema :
        match.decompose()


    # All the "property" and "typeof" attributes are suppressed as well as rel="any rdf vocab"
    prop = soup.find_all(property=True)
    typeof = soup.find_all(typeof=True)
    dc = soup.find_all(rel=re.compile("dcterms*"))
    rdf = soup.find_all(rel=re.compile("rdf*"))
    foaf= soup.find_all(rel=re.compile("foaf*"))

    for match in prop :
        del match['property']

    for match in typeof :
        del match['typeof']

    for match in dc :
        del match['rel']

    for match in rdf :
        del match['rel']

    for match in foaf :
        del match['rel']
        
    with open(str(i)+'/isaw-papers-'+str(i)+'-ereaders.xhtml', 'w') as paper:
        paper.write(str(soup))<jupyter_output>/anaconda3/lib/python3.6/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system ("lxml"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.

The code that caused this warning is on line 193 of the file /anaconda3/lib/python3.6/runpy.py. To get rid of this warning, change code that looks like this:

 BeautifulSoup(YOUR_MARKUP})

to this:

 BeautifulSoup(YOUR_MARKUP, "lxml")

  markup_type=markup_type))
{'fraction_non_alphanumeric': 0.12266202696824706, 'fraction_numerical': 0.0078294910830796, 'mean_word_length': 3.197080291970803, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '146566', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 789, 'n_tokens_neox': 747, 'n_words': 224, 'path': '.ipynb_checkpoints/cleaning-epubs-simplified-checkpoint.ipynb', 'repo_name': 'FMezard/isaw-papers-epub', 'size': '2299'}