<jupyter_start><jupyter_text>Scrap Indeed<jupyter_code>import requests
import bs4
from bs4 import BeautifulSoup
import pandas as pd
import time


def extract_job_title_from_result(soup):
    jobs = []
    for div in soup.find_all(name="div", attrs={"class": "row"}):
        for a in div.find_all(name="a", attrs={"data-tn-element": "jobTitle"}):
            jobs.append(a["title"])
    return(jobs)


def extract_location_from_result(soup):
    locations = []
    for div in soup.find_all(name="div", attrs={"class": "row"}):
        location = div.find_all(name="div", attrs={"class": "location"})
        if len(location) > 0:
            for b in location:
                locations.append(b.text.strip())
        else:
            sec_try = div.find_all(name="span", attrs={"class": "location"})
            for span in sec_try:
                locations.append(span.text.strip())
    return(locations)


def extract_company_from_result(soup):
    companies = []
    for div in soup.find_all(name="div", attrs={"class": "row"}):
        company = div.find_all(name="span", attrs={"class": "company"})
        if len(company) > 0:
            for b in company:
                companies.append(b.text.strip())
        else:
            sec_try = div.find_all(
                name="a", attrs={"data-tn-element": "companyName"})
            if len(sec_try) > 0:
                for span in sec_try:
                    companies.append(span.text.strip())
            else:
                companies.append("Nothing_found")
    return(companies)


def extract_salary_from_result(soup):
    salaries = []
    for div in soup.find_all(name="div", attrs={"class": "row"}):
        salary = div.find_all(name="span", attrs={"class": "salary no-wrap"})
        if len(salary) > 0:
            for b in salary:
                salaries.append(b.text.strip())
        else:
            salaries.append("Nothing_found")
    return(salaries)


def extract_summary_from_result(soup):
    summaries = []
    spans = soup.findAll("div", attrs={"class": "summary"})
    for span in spans:
        summaries.append(span.text.strip())
    return(summaries)


def add_to_df(df, soup):
    # Run multiple extractions to add to df
    columns = ["job_title", "location", "company", "salary", "summary"]
    temp = pd.DataFrame(columns=columns)
    print(len(extract_job_title_from_result(soup)), len(extract_location_from_result(soup)), len(
        extract_company_from_result(soup)), len(extract_salary_from_result(soup)), len(extract_summary_from_result(soup)))
    funcs = [extract_job_title_from_result(soup), extract_location_from_result(soup), extract_company_from_result(
        soup), extract_salary_from_result(soup), extract_summary_from_result(soup)]
    for i, j in enumerate(funcs):
        temp[columns[i]] = j

    return pd.concat([df, temp])


max_results_per_city = 170
columns = ["job_title", "location", "company", "salary", "summary"]
sample_df = pd.DataFrame(columns=columns)

for start in range(0, max_results_per_city, 10):
    URL = "https://www.indeed.com/jobs?q=data+scientist&l=georgia&start=" + str(start)
    page = requests.get(URL)
    soup = BeautifulSoup(page.text, "html.parser")
    soup.prettify()
    sample_df = add_to_df(sample_df, soup)
    time.sleep(10)

from datetime import date
today = str(date.today()) # '2018-12-26'

sample_df.to_csv("jobs_Georgia_"+today+".csv", index=False)<jupyter_output>19 19 19 19 19
19 19 19 19 19
19 19 19 19 19
19 19 19 19 19
16 16 16 16 16
17 17 17 17 17
15 15 15 15 15
17 17 17 17 17
15 15 15 15 15
15 15 15 15 15
15 15 15 15 15
15 15 15 15 15
16 16 16 16 16
15 15 15 15 15
15 15 15 15 15
15 15 15 15 15
15 15 15 15 15<jupyter_text>Preview<jupyter_code>import pandas as pd
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 500)
df = pd.read_csv("jobs_Georgia_"+today+".csv")
df.head()<jupyter_output><empty_output><jupyter_text>Companies - Job Listing Counts<jupyter_code>df.company.value_counts()<jupyter_output><empty_output><jupyter_text>Job Title Counts<jupyter_code>df.job_title.value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>Location Counts<jupyter_code>df.location.value_counts()<jupyter_output><empty_output><jupyter_text>Salary Counts<jupyter_code>df.salary.value_counts()<jupyter_output><empty_output><jupyter_text>Groupby Job Titles & Companies<jupyter_code>df.groupby(['company', 'job_title']).sum()<jupyter_output><empty_output><jupyter_text>Groupby Salary & Companies<jupyter_code>df.groupby(['salary','company']).sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13592022544981575, 'fraction_numerical': 0.043572512464773464, 'mean_word_length': 3.8980891719745223, 'pattern_counts': {'":': 11, '<': 32, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '209251', 'license': 'MIT', 'n_tokens_mistral': 1799, 'n_tokens_neox': 1594, 'n_words': 328, 'path': 'job_finder_indeed.ipynb', 'repo_name': 'extrajp2014/Data-Mining-Indeed', 'size': '4613'}