<jupyter_start><jupyter_text>Must run the following commands in the appropriate conda env in order for this to work:>  \> conda install spacy  >  \> python -m spacy download en    (second one must be run as admin)Alternatively run the top command and then replace the cell below with:  `import spacy  import en_core_web_sm  nlp = en_core_web_sm.load()`<jupyter_code>import spacy
nlp = spacy.load('en')
# driver
def get_skill_map(skills_list):
    """
    Returns a dict mapping each skill in the skill_list
    to a list of course names associated with that skill.
    """
    # first get the links for all Heinz courses
    course_dict = get_course_links()
    course_descriptions = course_dict.copy()

    # then scrape the description text for each link
    for name, link in course_dict.items():
        course_descriptions[name] = get_course_description(link)
        
    # time for a little nlp (to lemmatize the descriptions)
    # we'll store the results in a new dictionary
    parsed_descriptions = course_descriptions.copy()
    for name, text in course_descriptions.items():
        parsed_text = nlp(text)
        parsed_tokens = []
        
        # remove all stopwords, punctuation, and spaces
        for token in parsed_text:
            lemma = token.lemma_.lower()
            if not (nlp.vocab[lemma].is_stop or token.pos_ == 'PUNCT' or token.pos_ == 'SPACE'):
                parsed_tokens.append(lemma)
        
        # finally add list to dictionary
        parsed_descriptions[name] = parsed_tokens
    
    
    # now we can iterate through the skills list and build our final dictionary
    # give each skill its own list
    skills_to_courses = { k : [] for k in skill_list }
    
    for skill in skills_list:
        # now check every course for that skill
        for course_name in parsed_descriptions.keys():
            # if that skill appears in the description, add to the map
            if skill.lower() in parsed_descriptions[course_name]:
                skills_to_courses[skill].append(course_name)
    
    # return the final mapping
    return skills_to_courses
def get_course_links():
    """
    Get a list of urls to Heinz College course description 
    pages, so we can scrape each of those iteratively.
    
    Params
    -------
    skill_list
        A list of all the skills searched for in these pages
    
    Return
    -------
    <dict> {course_name: course_link, ...}
        course_name: string containing course name
        course_link: string containing link to course description page
    """
    html_base = 'https://api.heinz.cmu.edu'
    html = urlopen('https://api.heinz.cmu.edu/courses_api/course_list/')
    soup = BeautifulSoup(html.read(), "lxml")
    
    names = []
    links = []
    
    # Each course link is contained in a <tr> element with class="clickable-row"
    course_rows = soup.findAll('tr', {'class': 'clickable-row'})
    for row in course_rows:
        td = row.findAll('td')
        num = td[0].string
        name = td[1].string
        names.append(num + ": " + name)
        links.append(html_base + row.get('data-href'))
    
    return dict(zip(names, links))
def get_course_description(link):
    # Retrieve the html
    html = urlopen(link)
    soup = BeautifulSoup(html.read(), "lxml")

    # Recover the description text
    text = ' '.join([p.get_text() for p in soup.find_all('p')])
    clean = re.sub(r'[^\w\s]', '', text)
    return clean
html = urlopen('https://api.heinz.cmu.edu/courses_api/course_list/')
soup = BeautifulSoup(html.read(), "lxml")
course_rows = bsyc.findAll('tr', {'class': 'clickable-row'})
course_links = get_course_links()
course_links['Statistical Reasoning']
html = urlopen(course_links['Statistical Reasoning'])
soup = BeautifulSoup(html.read(), "lxml")
html_base = 'https://api.heinz.cmu.edu'
html = urlopen('https://api.heinz.cmu.edu/courses_api/course_list/')
soup = BeautifulSoup(html.read(), "lxml")
names = []
links = []

# Each course link is contained in a <tr> element with class="clickable-row"
course_rows = soup.findAll('tr', {'class': 'clickable-row'})
course_rows[0].findAll('td')[0].string
t = get_course_description(course_links['Statistical Reasoning'])
t
text = ' '.join([p.get_text() for p in soup.find_all('p')])
text
# test it
test_list = ['Management', 'software', 'knowledge']
results = get_skill_map(test_list)
write_to_csv(results)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11249434133091897, 'fraction_numerical': 0.0009053870529651426, 'mean_word_length': 3.1337698783910195, 'pattern_counts': {'":': 1, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '302871', 'license': 'Naumen', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1364, 'n_words': 399, 'path': 'Heinz Course Scraping.ipynb', 'repo_name': 'bjrhindress/Heinz-Skill-Scraper', 'size': '4418'}