<jupyter_start><jupyter_text>Project Data Science Desired Skillsthis project will Answer the question of what are the most desired skills for Data Science in Egypt and UAE.* Extract data from Linked in Via webscrapping or Manually.* Perform data wrangling ( image recognition or NLP).* Segment the the job market based on skills.* Analyze the skill growth over time.<jupyter_code>#import needed libraries

from selenium import webdriver
from selenium.webdriver.common.by import By
import pandas as pd
from scrapy import selector
import time
import operator
from functools import reduce
import dateparser
import numpy as np
import matplotlib.pyplot as plt

import re
import string
from collections import Counter
import spacy
from spacy.vocab import Vocab
from spacy.tokenizer import Tokenizer
import en_core_web_sm
from nltk.stem import PorterStemmer

from bs4 import BeautifulSoup<jupyter_output><empty_output><jupyter_text>step 1 Importing Data  Selenium driver for chrome<jupyter_code># instantiate a webdriver
driver = webdriver.Chrome('C:\\Users\\mosta\\Downloads\\chromedriver.exe')
# user search keywards and location
#job, loc= input("please enter the desired job and location:").split()
#url= 'https://www.linkedin.com/jobs/search/?geoId=106155005&keywords={}&location={}'
# use chrome to get the desired url
driver.get('https://www.linkedin.com/jobs/search/?geoId=106155005&keywords=data%20scientist&location=Egypt')
# the job card container(link for job)
#full_card_link= driver.find_elements_by_class_name("result-card__full-card-link")
full_card_link= driver.find_elements_by_class_name("base-card__full-link")
# view the Selenium elment count
len(full_card_link)
# extract Href from full card link
link_list=[]
for element in full_card_link:
    link_list.append(element.get_attribute('href'))
# create list place holder for data
job_title=[]
company=[]
location=[]
posting_date=[]
applicants_count=[]
job_description=[]
skill_set=[]
job_seniority_level=[]
employment_type=[]
industry=[]
job_function=[]
# create a function to iterate over job links

def get_job(links):
    ''' function to import the required data (10 specifications) from the search url'''
    for link in range(len(links)):
        driver.get(links[link])
        job_title.append(driver.find_element_by_class_name("topcard__title").text)
        company.append(driver.find_element_by_class_name('topcard__flavor').text)
        location.append(driver.find_element_by_xpath('/html/body/main/section[1]/section[2]/div/div[1]/div/h4/div[1]/span[2]').text)
        posting_date.append(driver.find_element_by_xpath('/html/body/main/section[1]/section[2]/div/div[1]/div/h4/div[2]/span[1]').text)
        try:
            applicants_count.append(driver.find_element_by_xpath('//*[@id="main-content"]/section[1]/section[2]/div[1]/div[1]/h3[2]/span[2]').text)
        except:
            applicants_count.append('')
        job_description.append(driver.find_element_by_xpath('//*[@id="main-content"]/section[1]/section[3]/div/section/div').text)
        try:
            job_seniority_level.append(driver.find_element_by_xpath('//*[@id="main-content"]/section[1]/section[3]/ul/li[1]/span').text)
        except:
            job_seniority_level.append('')
        try:
            employment_type.append(driver.find_element_by_xpath('/html/body/main/section[1]/section[3]/ul/li[2]/span').text)
        except:
            employment_type.append('')            
        try:
            industry.append(driver.find_element_by_xpath('//*[@id="main-content"]/section[1]/section[3]/ul/li[4]/span').text)
        except:
            industry.append('')
        try:
            job_function.append(reduce(operator.concat, [(e.text+ ", ") for e in driver.find_elements_by_xpath('//*[@id="main-content"]/section[1]/section[3]/ul/li[3]/span')]))
        except:
            job_function.append('')
        try:
            skill_set.append(reduce(operator.concat, [(e.get_attribute('innerHTML')) for e in driver.find_element_by_xpath('//*[@id="main-content"]/section[1]/section[3]/div/section/div').find_elements_by_tag_name("li")]))
        except:
            skill_set.append('')
        time.sleep(15)

#use the function to import the data
get_job(link_list)
# convert lists acqquired to a dataframe
data= pd.DataFrame(list(zip(job_title, company, location, posting_date, applicants_count, job_description, skill_set, job_seniority_level,
                            employment_type, industry, job_function)),
                   columns=['job_title', 'company', 'location', 'posting_date', 'applicants_count', 'job_description', 'skill_set','job_seniority_level',
                            'employment_type', 'industry', 'job_function'])
# inspect the dataframe
data.info()
# view the df head
data.head()
# inspect duplicates
data[data.duplicated()]
# we will asssign it to a dates variable and put it in a list and apply changes to the values as shown below
dates=list(data['posting_date'])
# explore the job post date column
dates
# a funtion that transforms 'x ago format' to 'datetime' format
def date(x):
    return dateparser.parse(x).strftime("%Y-%m-%d")
# for an example
date('3 weeks ago')
# apply the fnction to a dates list
transformed_dates=list(map(date,dates))
# the transformed resullt
transformed_dates
# replace the new dates values with the old posting_date column values in the data frame
data['posting_date'] =np.array(transformed_dates)
# transforming the posting_date datatype into 'datetime' instead of 'object'
data['posting_date']=pd.to_datetime(data['posting_date'])
#the final result
data.head()
# inspect the output dataframe
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 24 entries, 0 to 23
Data columns (total 11 columns):
 #   Column               Non-Null Count  Dtype         
---  ------               --------------  -----         
 0   job_title            24 non-null     object        
 1   company              24 non-null     object        
 2   location             24 non-null     object        
 3   posting_date         24 non-null     datetime64[ns]
 4   applicants_count     24 non-null     object        
 5   job_description      24 non-null     object        
 6   skill_set            24 non-null     object        
 7   job_seniority_level  24 non-null     object        
 8   employment_type      24 non-null     object        
 9   industry             24 non-null     object        
 10  job_function         24 non-null     object        
dtypes: datetime64[ns](1), object(10)
memory usage: 2.2+ KB<jupyter_text>Observations* The dataset contains 23 rows and 11 columns.* The column data types are objects.* There is a no duplicated row in the dataset.* posting_date column was in the format of 'x days ago' and a string data type which we had to amend and transform it into a "%Y-%m-%d" format and a datetime datatype  Exploratory Data analysis<jupyter_code># check summary statistics
data.describe()
# checking applicat count for each company with a bar chart

plt.figure(figsize=(15,15))
plt.bar(data['company'],data['applicants_count'])<jupyter_output><empty_output><jupyter_text>which market skills are needed for a Data scientist title?<jupyter_code>data['skill_set']
# create a csv as a reference
data.to_csv('linkedin_jobs.csv')<jupyter_output><empty_output><jupyter_text>Part 2  Extracting Insights about skill_set using NLP<jupyter_code># declare a function to prepare skil_set column for nlp

def preprocess_text(text):
    text = text.replace('\n', ' ')
    text = BeautifulSoup(text, 'lxml').get_text()
    text = text.replace('/', ' ')
    text = re.sub(r'[^a-zA-Z ^0-9]', '', text)
    text = text.lower()
    text = re.sub(r'(x.[0-9])', '',text)
    return text
# apply preprocessing to the desired column
data['refined_skilss']= data.apply(lambda x: preprocess_text(x['skill_set']), axis=1)
# inspect the pre-processd column
data['refined_skilss']
# Intiation of the tokenizer
from spacy.tokenizer import Tokenizer
from spacy.lang.en import English

nlp = spacy.load("en_core_web_sm")


tokenizer = Tokenizer(nlp.vocab)


Stop_words = nlp.Defaults.stop_words<jupyter_output><empty_output><jupyter_text>[code for below text was referenced from this website](https://prrao87.github.io/blog/spacy/nlp/performance/2020/05/02/spacy-multiprocess.htmlRead-in-New-York-Times-Dataset)<jupyter_code># define lemmatizer function
def lemmatize(text):
    doc =nlp(text)
    lemma_list = [str(tok.lemma_).lower() for tok in doc
                 if tok.is_alpha and tok.text.lower() not in Stop_words]
    return lemma_list
data['lemma']= data['refined_skilss'].apply(lemmatize)
'''
tokens = []

for doc in tokenizer.pipe(data['refined_skilss'], batch_size=500):
    
    doc_tokens = []
    for token in doc:
        if (token.lemma_ not in Stop_words) & (token.text != ' '):
            doc_tokens.append(token.lemma)

    tokens.append(doc_tokens)

data['tokens'] = tokens
'''
             

# Use English stemmer.
#from nltk.stem.snowball import SnowballStemmer
#stemmer = SnowballStemmer("english")
#data['tokens'] = data['refined_skilss'].apply(lambda x: [stemmer.stem(x) for y in x]) # Stem every word.

data['lemma']
'''
function to split the required skills per token

from collections import Counter


p = Counter(" ".join(data['lemma']).split()).most_common(10)
rslt = pd.DataFrame(p, columns=['Word', 'Frequency'])
print(rslt)
'''<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12156862745098039, 'fraction_numerical': 0.014944356120826709, 'mean_word_length': 3.863917525773196, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '251439', 'license': 'MIT', 'n_tokens_mistral': 3080, 'n_tokens_neox': 2949, 'n_words': 745, 'path': 'Linked-in Job scrapper Revision 2.ipynb', 'repo_name': 'MMallah/Linked-in-Job-scrapper', 'size': '9435'}