<jupyter_start><jupyter_text>SCRAPING MORE INTERESTING INFORMATION<jupyter_code>url = 'https://en.wikipedia.org/wiki/List_of_European_countries_by_life_expectancy'
html = requests.get(url).content # or .text
soup = BeautifulSoup(html, "lxml")
soup # shows all messy content
table = soup.find_all('table',{'class':'sortable wikitable'})[0] # dictionary is the attribute
table
table.find_all('tr') # gives all info
table.find_all('tr')[0].text # 0 or 1 gives different data rows
table.find_all('tr')[0].text.strip() # simplestrip does it on right strip (rstrip) and left strip (lstrip)
rows = table.find_all('tr') # getting into next level
rows = [row.text.strip().split("\n") for row in rows] # removes the \n
rows
rows[0]
colnames = rows[0]
data = rows[1:]
import pandas as pd
df = pd.DataFrame(data)
df
df.columns = ['Rank', 'Blank', 'Country', 'LE1', 'LE2'] # second column was blank so dropped
df = df.drop('Blank', axis =1)
df.head() # make one column and drop it when creating column with both info. make it a list and then map it lol
df.dtypes
df = df.astype({'LE1':'float'}, errors = 'ignore')
df = df.astype({'LE2':'float'}, errors = 'ignore')
df.dtypes
def clean(x, y):
    if x is None or x is "" and type(y) is float:
        return y
    else:
        return x
df['LE'] = list(map(clean, df['LE1'], df['LE2']))
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15498154981549817, 'fraction_numerical': 0.01107011070110701, 'mean_word_length': 4.87012987012987, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '618447', 'license': 'MIT', 'n_tokens_mistral': 486, 'n_tokens_neox': 471, 'n_words': 149, 'path': 'module-1/API-Scavenger/your-code/.ipynb_checkpoints/Web Scraping-checkpoint.ipynb', 'repo_name': 'gracegm/daft-miami-1019-labs', 'size': '1355'}