<jupyter_start><jupyter_text>Mechatronics Option Collect course link texts (for driver to click on)<jupyter_code>start = page_soup.find("div", {"id": "stack"}).find("p", text="Core Courses for Mechatronics Option:")
start
option_core = start.findNext("p")
option_core
option_electives = option_core.findNext("p").findNext("p")
option_electives
containers = option_core.findAll("a") + option_electives.findAll("a")
link_texts = [container.text.strip() for container in containers if container.text.strip() != 'ECE 2701']
link_texts
len(link_texts)<jupyter_output><empty_output><jupyter_text>Automation script to scrape all course info<jupyter_code>from selenium.common.exceptions import NoSuchElementException

course_codes = []
course_names = []
course_descs = []
counter = 0
    
for link_text in link_texts:
    #go to course page
    try:
        link = driver.find_element_by_link_text(link_text)
    except NoSuchElementException:
        print("no link for {}".format(link_text))
        continue

    time.sleep(2)
    link.click()    
    time.sleep(2)
    
    #scrape course info
    page_soup = soup(driver.page_source, 'lxml')
    container = page_soup.find("table", {"class": "ugradcal large-only"})
    course_code = container.find("th", {"abbr": "Course Code"}).text
    course_name = container.find("th", {"abbr": "Course Dscription"}).text
    course_desc = container.find("course_description").text.strip()
    
    course_codes.append(course_code)
    course_names.append(course_name)
    course_descs.append(course_desc)
    
    print("Scraped ", course_codes[-1])
    counter += 1
    
    #go to course list page
    driver.back()
    time.sleep(2)
        
print("Finished scraping {} courses".format(counter))
course_codes
course_names
course_descs
course_descs = [desc.replace("\xa0", " ") for desc in course_descs]
import pandas as pd

df = pd.DataFrame({
    "Course Number": course_codes,
    "Course Name": course_names,
    "Course Description": course_descs
})

df
df.to_csv('UNewBrunswick_MechEng_Mechatronics_Option_Core_and_Electives_Courses.csv', index = False)<jupyter_output><empty_output><jupyter_text>Biomedical Option Collect course link texts (for driver to click on)<jupyter_code>page_soup = soup(driver.page_source, 'lxml')
start = page_soup.find("div", {"id": "stack"}).find("p", text="Requirements:")
start
option_core = start.findNext("p")
option_core
option_electives1 = option_core.findNext("p").findNext("p")
option_electives1
option_electives2 = option_electives1.findNext("p").findNext("p")
option_electives2
containers = option_core.findAll("a") + option_electives1.findAll("a") + option_electives2.findAll("a")
link_texts = [container.text.strip() for container in containers]
link_texts
len(link_texts)<jupyter_output><empty_output><jupyter_text>Automation script to scrape all course info<jupyter_code>from selenium.common.exceptions import NoSuchElementException

course_codes = []
course_names = []
course_descs = []
counter = 0
    
for link_text in link_texts:
    #go to course page
    try:
        link = driver.find_element_by_link_text(link_text)
    except NoSuchElementException:
        print("no link for {}".format(link_text))
        continue

    time.sleep(2)
    link.click()    
    time.sleep(2)
    
    #scrape course info
    page_soup = soup(driver.page_source, 'lxml')
    container = page_soup.find("table", {"class": "ugradcal large-only"})
    course_code = container.find("th", {"abbr": "Course Code"}).text
    course_name = container.find("th", {"abbr": "Course Dscription"}).text
    course_desc = container.find("course_description").text.strip()
    
    course_codes.append(course_code)
    course_names.append(course_name)
    course_descs.append(course_desc)
    
    print("Scraped ", course_codes[-1])
    counter += 1
    
    #go to course list page
    driver.back()
    time.sleep(2)
        
print("Finished scraping {} courses".format(counter))
course_codes
course_names
course_descs
course_descs = [desc.replace("\xa0", " ") for desc in course_descs]
import pandas as pd

df = pd.DataFrame({
    "Course Number": course_codes,
    "Course Name": course_names,
    "Course Description": course_descs
})

df
df.to_csv('UNewBrunswick_MechEng_Biomedical_Option_Core_and_Electives_Courses.csv', index = False)
driver.quit()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12482758620689655, 'fraction_numerical': 0.005747126436781609, 'mean_word_length': 4.8324396782841825, 'pattern_counts': {'":': 14, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '261021', 'license': 'MIT', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1428, 'n_words': 287, 'path': 'Web-Scraping Scripts and Data/Accredited Canadian English Undergrad MechEng Programs/UNewBrunswick/WS_UNewBrunswick_MechEng_Options_Core_and_Electives_(AllYears).ipynb', 'repo_name': 'JiaShengJerryLu/Keyword-Matching-for-Canadian-Mechanical-Engineering-Programs', 'size': '4350'}