<jupyter_start><jupyter_text>Exploring indeed.com page elements<jupyter_code>def get_url(position, location):
  """Generate a url  from position and location """
  template = "https://www.indeed.com/jobs?q={}&l={}"
  url = template.format(position, location)
  return url
url = get_url('data scientist', 'bentonville ar')
response = requests.get(url)
response
response.reason
soup = BeautifulSoup(response.text, 'html.parser')
cards = soup.find_all('div', 'jobsearch-SerpJobCard')
len(cards)
card = cards[0]
atag = card.h2.a
job_title = atag.get('title')
job_url = 'https://www.indeed.com' + atag.get('href')
company = card.find('span', 'company').text.strip()
job_location = card.find('div', 'recJobLoc').get('data-rc-loc')
summary = card.find('div', 'summary').text.strip()
date = card.find('span', 'date').text
today = datetime.today().strftime('%Y-%m-%d')
try: 
  card.find('span', 'salarText').text.strip()
except AttributeError:
  job_salary = ''<jupyter_output><empty_output><jupyter_text>Generalizing the model<jupyter_code>def get_record(card):
  atag = card.h2.a
  job_title = atag.get('title')
  job_url = 'https://www.indeed.com' + atag.get('href')
  company = card.find('span', 'company').text.strip()
  job_location = card.find('div', 'recJobLoc').get('data-rc-loc')
  job_summary = card.find('div', 'summary').text.strip()
  post_date = card.find('span', 'date').text
  today = datetime.today().strftime('%Y-%m-%d')
  try: 
    card.find('span', 'salarText').text.strip()
  except AttributeError:
    job_salary = '' 
  
  record = {job_title, company, job_location, post_date, today, job_summary, job_salary, job_url}

  return record
records = []

for card in cards: 
  record = get_record(card)
  records.append(record)
records[0]<jupyter_output><empty_output><jupyter_text>Getting every page<jupyter_code>while True: 
  try: 
    url = 'https://www.indeed.com' + soup.find('a', {'aria-label': 'Next'}).get('href')
  except AttributeError:
    break

  response = requests.get(url)
  soup = BeautifulSoup(response.text, 'html.parser')
  cards = soup.find_all('div', 'jobsearch-SerpJobCard')

  for card in cards:
    record = get_record(card)
    records.append(record)<jupyter_output><empty_output><jupyter_text>Putting it all together<jupyter_code>def get_url(position, location):
    """Generate url from position and location"""
    template = 'https://www.indeed.com/jobs?q={}&l={}'
    position = position.replace(' ', '+')
    location = location.replace(' ', '+')
    url = template.format(position, location)
    return url


def get_record(card):
    """Extract job data from a single record"""
    
    job_title = card.h2.a.get('title')
    company = card.find('span', 'company').text.strip()
    job_location = card.find('div', 'recJobLoc').get('data-rc-loc')
    post_date = card.find('span', 'date').text
    today = datetime.today().strftime('%Y-%m-%d')
    summary = card.find('div', 'summary').text.strip().replace('\n', ' ')
    job_url = 'https://www.indeed.com' + card.h2.a.get('href')

    # this does not exists for all jobs, so handle the exceptions
    salary_tag = card.find('span', 'salaryText')
    if salary_tag:
        salary = salary_tag.text.strip()
    else:
        salary = ''  
        
    record = (job_title, company, job_location, post_date, today, summary, salary, job_url)
    return record


def main(position, location):
    """Run the main program routine"""
    records = []
    url = get_url(position, location)
    
    # extract the job data
    while True:
        response = requests.get(url)
        soup = BeautifulSoup(response.text, 'html.parser')
        cards = soup.find_all('div', 'jobsearch-SerpJobCard')
        for card in cards:
            record = get_record(card)
            records.append(record)
        try:
            url = 'https://www.indeed.com' + soup.find('a', {'aria-label': 'Next'}).get('href')
        except AttributeError:
            break
        
    # save the job data
    with open('results.csv', 'w', newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(['JobTitle', 'Company', 'Location', 'PostDate', 'ExtractDate', 'Summary', 'Salary', 'JobUrl'])
        writer.writerows(records)
main('data scientist', 'bentonville, ar')
import pandas as pd
df = pd.read_csv('results.csv')
df.head(1000)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16972056802565277, 'fraction_numerical': 0.0025194686211635364, 'mean_word_length': 3.934463276836158, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 7, 'lorem ipsum': 0, 'www.': 7, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '650024', 'license': 'MIT', 'n_tokens_mistral': 1488, 'n_tokens_neox': 1461, 'n_words': 299, 'path': 'notebooks/datasets/data/city_jobs/Indeed_Data_Scrape_EDA.ipynb', 'repo_name': 'sanjaykmenon/PT17_cityspire-d-ds', 'size': '4366'}