<jupyter_start><jupyter_text>ObjectiveExtract relevant sections from the job posting html files.<jupyter_code>import pathlib
from bs4 import BeautifulSoup 

cur_path = pathlib.Path()
data_path = cur_path.joinpath('data', 'html_job_postings')
html_files = data_path.iterdir() # iterator of html files<jupyter_output><empty_output><jupyter_text>Extract relevant texts from html<jupyter_code>def extract_text(html): 
    """
    Extract position title and job requirements from job positng html. 
    """
    with open(html, "r", encoding = "utf-8") as h:
        html_content = h.read()
    
    soup = BeautifulSoup(html_content)
    title = soup.find('title').text
    position, *location = title.split(" - ")

    if len(location) == 2:
        company, location = location 
    else:
        company, location = None, location[0]

    body = soup.find('body')
    if body.find('li'):
        jobreq = [bullet.text for bullet in body.find_all('li')]
    else:
        jobreq = [bullet.text for bullet in body.find_all('p')]
    
    return {'html': html,
            'position': position,
            'location': location,
            'company': company, 
            'jobreq': ' '.join(jobreq).strip('\n').replace('\n', '')}
    

data = []
for html in html_files:
    data.append(extract_text(html))

import pandas as pd 

df = pd.DataFrame(data)
print(df.info())
print(df.head())
df.loc[0, 'jobreq']<jupyter_output><empty_output><jupyter_text>Remove job postings unrelated to data scientist position<jupyter_code>df['position'] = df['position'].map(lambda p: p.lower())
print(df['position'].unique())
def keep_positions(df):
    """
    Keep positions that contain the word "data scientist" or are adjacent to data scientist. 
    """
    df['position'] = df['position'].map(lambda p: p.lower()) # lower case
    roles = ('data scientist', 'data analyst', 'statistical analyst', 
             'quantitative analyst', 'business analyst',
             'data science', 'data consultant', 
             'machine learning', 'data engineer', 'research analyst',
             'business insights analyst', 'quantitative',
             'analyst', 'intellligence', 'natural language processing', 'data',
             'research scientist')
    keep = [p for p in df['position'] if any(role in p for role in roles)]
    return df[[p in keep for p in df['position']]]
             
df = keep_positions(df)
print(df.info())<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 828 entries, 0 to 1336
Data columns (total 5 columns):
 #   Column    Non-Null Count  Dtype 
---  ------    --------------  ----- 
 0   html      828 non-null    object
 1   position  828 non-null    object
 2   location  828 non-null    object
 3   company   123 non-null    object
 4   jobreq    828 non-null    object
dtypes: object(5)
memory usage: 38.8+ KB
None<jupyter_text>Remove redundant job postings<jupyter_code>def uniques_only(df):
    """
    Identify html's of job postings whose job requirements are unique.
    """
    seen = []
    out = []
    for row in df.itertuples(index = True):
        if row.jobreq not in seen:
            out.append(row.html)
            seen.append(row.jobreq)
    return out
df = df[[h in uniques_only(df) for h in df['html']]]
print(df.info())
df.to_csv('output/data.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13486842105263158, 'fraction_numerical': 0.011961722488038277, 'mean_word_length': 3.0252707581227436, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '405903', 'license': 'Apache-2.0', 'n_tokens_mistral': 1070, 'n_tokens_neox': 1028, 'n_words': 292, 'path': '01-extract-text.ipynb', 'repo_name': 'amikami102/resume-job-posting-nlp-project', 'size': '3344'}