<jupyter_start><jupyter_text>Berlin tech job market - analysis and machine learningObjective: 1) Analyse word clusters in job title and job text based on distinct job_class- Data Scientist- Data Analyst- Data Engineer2) Train a SVM machine learning model to recognize distinct job titles3) Analyse and 3D visualisation on how distinct the job titles are used in the Berlin tech job market using PCA and t-SNE<jupyter_code># import statements
import time
import nltk
import pyLDAvis.gensim
import pyLDAvis.sklearn

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import plotly.express as px
import plotly.graph_objects as go

# sklear for machine learning model
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import svm
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE

from gensim import corpora, models

%matplotlib inline

nltk.download('wordnet')
# load pre-processed data set
df = pd.read_json('99_data/200810_preprocessed_df.json')
count = df[['job_class', 'ID']].groupby(['job_class'])['ID'].count()
fig = px.bar(df, x=count.index.values, y=count, 
             title="count jobs per job_class",
            labels={'x':'job_class','y':'count'})
fig.show()
company = df.groupby(['company_name'])['ID'].count().sort_values(ascending=False)
fig = px.bar(df, x=company.index.values, y=company, 
             title="count jobs per company",
            labels={'x':'company_name','y':'count'})
fig.show()
seniority = df.groupby(['seniority_level'])['ID'].count().sort_values(ascending=False)
seniority = seniority.drop(index='Keine Angabe')
fig = px.bar(df, x=seniority.index.values, y=seniority, 
             title="count jobs per seniority_level",
            labels={'x':'seniority_level','y':'count'})
fig.show()<jupyter_output><empty_output><jupyter_text>Train-test split the data set<jupyter_code># Generating Labels (Y)
# 0= Data Analyst, 1= Data Engineer, 2= Data Scientist
encoder = LabelEncoder()
y = encoder.fit_transform(df.job_class)
y_labels= pd.DataFrame(y)
df = pd.concat([df, y_labels], axis=1, sort=False)
df.rename(columns={0:'y_labels'}, inplace=True)

# train test split
x_train, x_test, y_train, y_test = train_test_split(df.loc[:, df.columns != 'y_labels'], df.y_labels, test_size=0.3, random_state=43)

x_train.shape, x_test.shape, y_train.shape, y_test.shape<jupyter_output><empty_output><jupyter_text>1) Analyse word clusters in job title and job text based on distinct job_class<jupyter_code># Generating Feature Vectors (X) with tfidf
# max features is the number of words from the text field that will be used to tfidf
v = TfidfVectorizer(max_features=200)
x = df['text_merge']
x1 = x_train['text_merge']
x2 = x_test['text_merge']
v.fit(x)
v.fit(x1)
v.fit(x2)
# this is the tfidf vector for data analysis
x_tfidf = v.transform(x)
# these are the x data, to use in the machine learning
x_train_tfidf = v.transform(x1) 
x_test_tfidf = v.transform(x2) 
x_tfidf, x_train_tfidf, x_test_tfidf

# create tfidf dataframe for all job_classes
tfidf_analysis_job_class = pd.DataFrame(x_tfidf.todense(), columns=v.vocabulary_, index=df.job_class).groupby('job_class').max()
tfidf_analysis_job_class
# most important words for Data Analyst
tfidf_analyst=tfidf_analysis_job_class.loc['Data Analyst'].sort_values(ascending=False)
# most important words for Data Scientist
tfidf_scientist=tfidf_analysis_job_class.loc['Data Scientist'].sort_values(ascending=False)
# most important words for Data Engineer
tfidf_engineer=tfidf_analysis_job_class.loc['Data Engineer'].sort_values(ascending=False)
# concat all 3 data frames
tfidf_analysis = pd.concat([tfidf_analyst, tfidf_scientist,tfidf_engineer], axis=1, sort=False)
tfidf_analysis.head(50)
# grouped bar chart
fig = go.Figure(data=[
    go.Bar(name='Data Analyst', x=tfidf_analysis.head(20).index.values, y=tfidf_analysis['Data Analyst']),
    go.Bar(name='Data Scientist', x=tfidf_analysis.head(20).index.values, y=tfidf_analysis['Data Scientist']),
    go.Bar(name='Data Engineer', x=tfidf_analysis.head(20).index.values, y=tfidf_analysis['Data Engineer'])
])
# Change the bar mode
fig.update_layout(barmode='group',title="tfidf score for 20 most common & unique words")
fig.show()
#heatmap
fig = go.Figure(data=go.Heatmap(
        z=[tfidf_analysis['Data Analyst'], tfidf_analysis['Data Scientist'], tfidf_analysis['Data Engineer']],
        x=tfidf_analysis.head(20).index.values,
        y=tfidf_analysis.columns,
        colorscale='Blues',
        hoverongaps = False))

fig.update_layout(title='Most important words in TFIDF vector analysis per job_class')

fig.show()
# feature extraction and filtering of words
job_features = corpora.Dictionary(df.lemma_text) 
job_features.filter_extremes(no_below=3,no_above=0.5)
#word count vector for each tweet for each of its words freq for the whole corpa 
bow_corpus=[job_features.doc2bow(doc) for doc in df.lemma_text]
#increase the frequency weight of unique words compared to highly frequent non-unique words 
tfidf=models.TfidfModel(bow_corpus)
tfidf_corpus=tfidf[bow_corpus]
lda_model=models.ldamodel.LdaModel(corpus=tfidf_corpus, num_topics=5, 
                                   id2word=job_features, 
                                   passes=50, 
                                   alpha=1, 
                                   eta='auto')
#take a look at the different topics 
for idx, topic in lda_model.print_topics(-1):
    print(f'Topic: {idx} \nWords: {topic}')
pyLDAvis.display(lda_data) #for notebook
# pyLDAvis.show(lda_data) #for browser
# define data frames for all 3 job_classes
df_analyst = df.set_index('job_class').loc[['Data Analyst'],['lemma_text']]
df_scientist = df.set_index('job_class').loc[['Data Scientist'],['lemma_text']]
df_engineer = df.set_index('job_class').loc[['Data Engineer'],['lemma_text']]
def LDA_topic_viz(data, job_title, num_topics):
    # feature extraction and filtering of words
    job_features = corpora.Dictionary(data)
    job_features.filter_extremes(no_below=3,no_above=0.5)
    #word count vector for each tweet for each of its words freq for the whole corpa 
    bow_corpus=[job_features.doc2bow(doc) for doc in df.lemma_text]
    time.sleep(10)
    #increase the frequency weight of unique words compared to highly frequent non-unique words 
    tfidf=models.TfidfModel(bow_corpus)
    tfidf_corpus=tfidf[bow_corpus]
    lda_model=models.ldamodel.LdaModel(corpus=tfidf_corpus, num_topics=num_topics, 
                                   id2word=job_features, 
                                   passes=50, 
                                   alpha=1, 
                                   eta='auto')
    #take a look at the different topics 
    print(f'These are the {num_topics} for {job_title}')
    for idx, topic in lda_model.print_topics(-1):
        print(f'Topic: {idx} \nWords: {topic}')
# call LDA function for 3 job_classes: Data Analyst
LDA_topic_viz(df_analyst.lemma_text,'data_analyst', 5)
# call LDA function for 3 job_classes: Data Scientist
LDA_topic_viz(df_scientist.lemma_text,'data_scientist', 5)

# call LDA function for 3 job_classes: Data Engineer
LDA_topic_viz(df_engineer.lemma_text,'data_engineer', 5)<jupyter_output>These are the 5 for data_engineer
Topic: 0 
Words: 0.008*"linux" + 0.007*"ein" + 0.007*"deutsch" + 0.007*"oder" + 0.007*"server" + 0.007*"campusjäg" + 0.006*"unser" + 0.006*"anzeigenwenig" + 0.006*"sowi" + 0.005*"instal"
Topic: 1 
Words: 0.005*"zalando" + 0.004*"discount" + 0.004*"tech" + 0.004*"impact" + 0.004*"machin" + 0.004*"need" + 0.004*"pipelin" + 0.004*"drive" + 0.003*"scale" + 0.003*"insight"
Topic: 2 
Words: 0.004*"forward" + 0.004*"learn" + 0.004*"report" + 0.004*"deliveri" + 0.004*"hero" + 0.003*"amazon" + 0.003*"individu" + 0.003*"gender" + 0.003*"friend" + 0.003*"exist"
Topic: 3 
Words: 0.005*"project" + 0.005*"digit" + 0.005*"train" + 0.004*"consult" + 0.004*"analysi" + 0.004*"profession" + 0.004*"research" + 0.004*"cours" + 0.004*"program" + 0.003*"ensur"
Topic: 4 
Words: 0.008*"bank" + 0.005*"senior" + 0.005*"monitor" + 0.005*"devop" + 0.005*"test" + 0.005*"kubernet" + 0.004*"modern" + 0.004*"energi" + 0.004*"invest" + 0.004*"databas"<jupyter_text>2) Train a SVM machine learning model to recognize distinct job titles<jupyter_code>s = svm.LinearSVC(C=1.0)
x_train_tfidf.shape, x_test_tfidf.shape, y_train.shape, y_test.shape
# fit the training dataset on the classifier
s.fit(x_train_tfidf, y_train)# predict the labels on validation dataset
# make predictions using the trained model
y_pred_train = s.predict(x_train_tfidf)
y_pred_test = s.predict(x_test_tfidf)

# Use accuracy_score function to get the accuracy
print("SVM Train Accuracy Score -> ",accuracy_score(y_pred_train, y_train)*100, "\n",
     "SVM Test Accuracy Score -> ",accuracy_score(y_pred_test, y_test)*100)

# confusion matrix training data
sns.heatmap(confusion_matrix(y_train,y_pred_train),annot=True,fmt='3.0f',cmap=sns.cubehelix_palette(8, start=.5, rot=-.75, as_cmap=True))
plt.title('Confusion_matrix for Training Data', y=1.05, size=15)

# confusion matrix test data
sns.heatmap(confusion_matrix(y_test,y_pred_test),annot=True,fmt='3.0f',cmap=sns.cubehelix_palette(8, start=.5, rot=-.75, as_cmap=True))
plt.title('Confusion_matrix for Test Data', y=1.05, size=15)<jupyter_output><empty_output><jupyter_text>3) Analyse and 3D visualisation using PCA and t-SNE<jupyter_code># first reduce dimensionality with pca to 50 before feeding to t-sne

pca = PCA(n_components= 50)
x_train_pca = pca.fit_transform(x_train_tfidf.toarray()) # transform sparse to dense<jupyter_output><empty_output><jupyter_text>2D visualisation<jupyter_code>mod = TSNE(n_components=2, verbose=0, perplexity=50, n_iter=5000, learning_rate=250)
title = "t-SNE decomposition" 
        
# Fit and transform the features
principal_components = mod.fit_transform(x_train_pca)
    
# Put them into a dataframe
df_features = pd.DataFrame(data=principal_components, columns=['PC1', 'PC2'])
    
# Now we have to paste each row's label and its meaning
# Convert labels array to df
df_labels = pd.DataFrame(data=y_train).rename(columns={'y_labels':'label'}).reset_index()
    
df_full = pd.concat([df_features, df_labels], axis=1)
df_full['label'] = df_full['label'].astype(str)

# Get labels name
category_names = {"0": 'Data Analyst',"1": 'Data Engineer',"2": 'Data Scientist'}

 # And map labels
df_full['label_name'] = df_full['label']
df_full = df_full.replace({'label_name':category_names})

# Plot
plt.figure(figsize=(10,10))
    
# 2D visualisation
fig = px.scatter(df_full, x="PC1", y="PC2", color='label_name')
fig.update_layout(title=title,
                  yaxis_zeroline=False, xaxis_zeroline=False)
fig.show()<jupyter_output><empty_output><jupyter_text>3D visualisation<jupyter_code>mod = TSNE(n_components=3, verbose=0, perplexity=50, n_iter=5000, learning_rate=250)
title = "t-SNE decomposition" 
        
# Fit and transform the features
principal_components = mod.fit_transform(x_train_pca)
    
# Put them into a dataframe
df_features = pd.DataFrame(data=principal_components, columns=['PC1', 'PC2', 'PC3'])
    
# Now we have to paste each row's label and its meaning
# Convert labels array to df
df_labels = pd.DataFrame(data=y_train).rename(columns={'y_labels':'label'}).reset_index()
    
df_full = pd.concat([df_features, df_labels], axis=1)
df_full['label'] = df_full['label'].astype(str)

# Get labels name
category_names = {"0": 'Data Analyst',"1": 'Data Engineer',"2": 'Data Scientist'}

 # And map labels
df_full['label_name'] = df_full['label']
df_full = df_full.replace({'label_name':category_names})
# 3D visualisation

fig = px.scatter_3d(df_full, x='PC1', y='PC2', z='PC3',
              color='label_name', title="t-SNE decomposition")
fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12849021902691857, 'fraction_numerical': 0.030513292091623474, 'mean_word_length': 5.122313203684749, 'pattern_counts': {'":': 6, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '78638', 'license': 'MIT', 'n_tokens_mistral': 4532, 'n_tokens_neox': 4207, 'n_words': 949, 'path': '.ipynb_checkpoints/job_data_machine_learning-checkpoint.ipynb', 'repo_name': 'SabFrAlphin/project_job_skill_screener_DATA_ML', 'size': '11962'}