<jupyter_start><jupyter_text>I was trying to parse all the urls of the plasitizers' properties on this website:https://app.knovel.com/web/toc.v/cid:kpPDEE0001/viewerType:toc//root_slug:plasticizers-database-2nd?&hierarchy=kn0V68XPV1 So that we can download the actrual files. But it is not as easy as I thought....  This code works for normal website...<jupyter_code>from bs4 import BeautifulSoup
import urllib3
import re
import certifi

# reading url and pass to html_page
http = urllib3.PoolManager()
html_page = http.request('GET',"http://depts.washington.edu/chem/facilserv/nmr/av300.html")

#pass to BS4
soup = BeautifulSoup(html_page.data)
# print (soup.prettify())
for link in soup.findAll('a', attrs={'href': re.compile("^http://")}):
    print (link.get('href'))<jupyter_output><empty_output><jupyter_text>It seems that they hide the data somewhere else. I can't find any names of plasticizers in the html code.<jupyter_code>from bs4 import BeautifulSoup
import urllib3
import re
import certifi

# reading url and pass to html_page
http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',ca_certs=certifi.where())
html_page = http.request('GET',"https://app.knovel.com/web/toc.v/cid:kpPDEE0001/viewerType:toc//root_slug:plasticizers-database-2nd?&hierarchy=kn0V68XPV1")

#pass to BS4
soup = BeautifulSoup(html_page.data)
print (soup.prettify())
for link in soup.findAll('a', attrs={'href': re.compile("^http://")}):
    print (link.get('href'))<jupyter_output><empty_output><jupyter_text>parsing from a single html<jupyter_code>from bs4 import BeautifulSoup
import requests
import pandas as pd
import csv


with open('97.html') as html_file:
    soup = BeautifulSoup (html_file, 'lxml')

chem = soup.find('table')
chem_name = chem.tbody.tr.b.text # name of the chemical 
chem_info = chem.tbody.tr.td # first level of information 


list_temp=[]

for div in chem_info.find_all('div'): #2nd level
    temp =  div.dl.dd
    for table in temp.find_all('table'): #3rd level 
        temp_1 = table.tbody.tr
#         list_temp = []
        for td in temp_1.find_all('td'): #4rd level -- actrual contents
            list_temp.append(td.text)
            
temp_properties ={}

for t_in in range(0, len(list_temp), 2):
        temp_properties[list_temp[t_in]] = list_temp[t_in + 1]

print(temp_properties)  

# with open("file_data.csv", "w+") as prop:
#     for row in temp_properties.items():
#         prop.write(row[0]+ " " + row[1] + "\n")

        # putting actual contents in to df
#         temp_properties = {list_temp[0]:list_temp[1]}
#         print (temp_properties)

### at the 4rd level there are two variables: names of the properties and their numbers
### How to put them into pandas dataframe???

with open("file_data.csv", "w+") as prop:
    for row in temp_properties.items():
        prop.write(row[0]+ " " + row[1] + "\n")
with open("file_data2.csv", "w+") as prop:
    for row in temp_properties.items():
        prop.write("\""+row[0]+ "\", \"" + row[1] + "\"\n")
with open("file_data.txt", "w+") as prop:
    for row in temp_properties.items():
        prop.write(row[0]+ " " + row[1] + "\n")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14213036565977744, 'fraction_numerical': 0.016534181240063592, 'mean_word_length': 4.2, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '86625', 'license': 'MIT', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1112, 'n_words': 280, 'path': 'notebooks/Trials-parsing.ipynb', 'repo_name': 'schaber/direct_capstone', 'size': '3145'}