<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as uReq

def get_raw_html_content(url):
  client = uReq(url)
  raw_content = client.read()
  client.close()
  return raw_content

# get html content
html_content = get_raw_html_content('https://www.classcentral.com/subject/data-science')

# load html content in bs4
page_soup = soup(html_content, 'html.parser')

# ratings should be always floats (star rating)
def rating_value(rating):
  try:
    return float(rating)
  except ValueError:
    return -1

def get_text_from_elements(elements):
  results = []
  for element in elements:
    # strip the whitespaces
    results.append(element.text.strip())
  return results
    

# load courses
course_elements = page_soup.findAll("span",{'class' : 'text-1 weight-semi line-tight'})
courses = get_text_from_elements(course_elements)


# load provider
provider_elements = page_soup.findAll('a',href=True, attrs={'class':'color-charcoal italic'})
providers = get_text_from_elements(provider_elements)


# load ratings
ratings = []
rating_elements = page_soup.findAll('div', attrs={'class':'col border-box text-center nowrap row large-up-text-right padding-horz-small push'})
for rating in rating_elements:
  value = rating.find('span', attrs={'class':'xlarge-up-hidden color-charcoal text-center'})
  ratings.append(rating_value(value.text.strip()))


# export data with pandas
df = pd.DataFrame({'course':courses,'ratings':ratings,'provider':providers})
print(df)
df.to_csv('classcentral_ds_courses.csv')<jupyter_output>course  ratings     provider
0                                       R Programming      2.8     Coursera
1                        The Data Scientist’s Toolbox      3.3     Coursera
2                        Computational Social Science      4.8     Coursera
3                           Getting and Cleaning Data      3.5     Coursera
4                                  The Analytics Edge      4.7          edX
5                           Exploratory Data Analysis      3.9     Coursera
6                               Become a Data Analyst      4.5      Udacity
7                            Introduction to Big Data      2.7     Coursera
8              Introduction to Data Science in Python      2.4     Coursera
9                             Python for Data Science      4.4          edX
10                   Mastering Data Analysis in Excel      1.8     Coursera
11                     A Crash Course in Data Science      3.5     Coursera
12          [...]
{'fraction_non_alphanumeric': 0.0828125, 'fraction_numerical': 0.0171875, 'mean_word_length': 2.304516129032258, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '369705', 'license': 'MIT', 'n_tokens_mistral': 755, 'n_tokens_neox': 736, 'n_words': 202, 'path': 'bs_webscrape_example.ipynb', 'repo_name': 'lukas-weiss/webscrape_example', 'size': '2560'}