<jupyter_start><jupyter_text>Resume Job Postings3. Finding clusters of jobs<jupyter_code># Setup
import pandas as pd
import numpy as np

from sklearn.decomposition import TruncatedSVD
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.metrics import silhouette_samples
from sklearn.preprocessing import Normalizer

from tqdm.notebook import tqdm
import matplotlib.cm as cm
from wordcloud import WordCloud
from sklearn.feature_extraction.text import TfidfVectorizer
import matplotlib.pyplot as plt

%matplotlib inline
# Load pickle file from step 2
most_similar_df = pd.read_pickle('step2_df.pk')
most_similar_df.head()
# Use bullets to analyze skill requirements
bullet_points = [item for sublist in most_similar_df['bullets'] for item in sublist]
len(bullet_points)
# Remove stop words
bullet_vectorizer = TfidfVectorizer(stop_words='english')
tfidf_skills = bullet_vectorizer.fit_transform(bullet_points)
tfidf_skills.shape
# Dimension reduction
dim = 500 # max value of (tfidf_skills.shape[1] - 1)
svd = TruncatedSVD(n_components=dim)

lsa = svd.fit_transform(tfidf_skills)
# total variance explained with 500 dimensions instead of 2647
svd.explained_variance_ratio_.sum()
# How many dimensions explain 50% of the variance
(svd.explained_variance_ratio_.cumsum() <= 0.5).sum()
# Check how evolve the variance with all dimensions
plt.plot(range(svd.explained_variance_ratio_.shape[0]), svd.explained_variance_ratio_.cumsum())
svd.explained_variance_ratio_.cumsum()[:10]
# Check variance explained with 100 dimensions
svd.explained_variance_ratio_[:100].sum()
# Normalize the results with only 100 components
svd = TruncatedSVD(n_components=100)
lsa = svd.fit_transform(tfidf_skills)
norm = Normalizer().fit_transform(lsa) # Normalization makes KMeans behave as spherical for better results
# Cluster using KMeans. Identify number of clusters
wss = []
silhouette = []
clusters = range(2, 51)
for c in tqdm(clusters):
    km = KMeans(n_clusters=c, random_state=42)
    km.fit(norm)
    # sklearn kmeans wss is negative
    wss.append(-km.score(norm))
    silhouette.append(silhouette_score(norm, km.labels_))
metrics = pd.DataFrame({'wss': wss, 'silhouette': silhouette})
# Plot number of cluster
metrics['wss'].plot()
plt.xlabel('number of clusters')
plt.ylabel('WSS')
_ = plt.axvline(6, c='r')
# Plot silhouette
metrics['silhouette'].plot()
plt.xlabel('number of clusters')
plt.ylabel('Silhouette')
_ = plt.axvline(6, c='r')
def silhouette_analysis(k):
    """Plot the silhouette of k number of clusters.
    """
    km = KMeans(n_clusters=k, random_state=42)
    km.fit(norm)
    cluster_labels = km.labels_
    # The silhouette_score gives the average value for all the samples.
    # This gives a perspective into the density and separation of the formed
    # clusters
    silhouette_avg = silhouette_score(norm, cluster_labels)
    print("For n_clusters =", k, "The average silhouette_score is :", silhouette_avg)
    
    # Compute the silhouette scores for each sample
    sample_silhouette_values = silhouette_samples(norm, cluster_labels)

    y_lower = 10
    for i in range(k):
        # Aggregate the silhouette scores for samples belonging to
        # cluster i, and sort them
        ith_cluster_silhouette_values = sample_silhouette_values[cluster_labels == i]
        
        ith_cluster_silhouette_values.sort()
        
        size_cluster_i = ith_cluster_silhouette_values.shape[0]
        y_upper = y_lower + size_cluster_i

        color = cm.nipy_spectral(float(i) / k)
        plt.fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_silhouette_values,
                          facecolor=color, edgecolor=color, alpha=0.7)
        
        # Label the silhouette plots with their cluster numbers at the middle
        plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
        
        # Compute the new y_lower for next plot
        y_lower = y_upper + 10 # 10 for the 0 samples

    plt.title("The silhouette plot for the various clusters.")
    plt.xlabel("The silhouette coefficient values")
    plt.ylabel("Cluster label")
    # The vertical line for average silhouette score of all the values
    plt.axvline(x=silhouette_avg, color="red", linestyle="--")
    plt.yticks([]) # Clear the yaxis labels / ticks
    plt.xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])
    plt.show()
# plot for 5 clusters
silhouette_analysis(5)
# plot for 6 clusters
silhouette_analysis(6)
clusters = 5
km = KMeans(n_clusters=clusters, random_state=42)
km.fit(norm)
cluster_labels_df = pd.DataFrame({'Cluster': km.labels_})
cluster_labels_df.head()
def rank_words_by_tfidf(indices, tfidf_matrix=tfidf_skills, word_list=bullet_vectorizer.get_feature_names()):
    """
    Takes indices for a tfidf matrix, sums the TFIDF across the matrix, and returns
    a pandas DataFrame with words ranked from the highest summed TFIDF values down to lowest.
    The indices argument is intended to be used to filter the DataFrame by kmeans clusters.
    """
    summed_tfidf = np.asarray(tfidf_matrix[indices].sum(axis=0))[0]
    data = {'Word': word_list, 'Summed TFIDF': summed_tfidf}
    return pd.DataFrame(data).sort_values('Summed TFIDF', ascending=False)
def cluster_to_image(df_cluster, max_words=15):
    indices = df_cluster.index.values
    # Get TFIDF-ranked words
    df_ranked_words = rank_words_by_tfidf(indices)[:max_words]

    # Create word-to-TFIDF value dictionary for the word cloud
    words_to_score = {word: score for word, score in df_ranked_words[:max_words].values if score != 0}
    
    # Generate the word cloud
    cloud_generator = WordCloud(background_color='white', color_func=_color_func, random_state=1)
    wordcloud_image = cloud_generator.fit_words(words_to_score)
    
    return wordcloud_image, words_to_score

def _color_func(*args, **kwargs):
    # a small function to change the color of the word clouds
    return np.random.choice(['black', 'blue', 'teal', 'purple', 'brown'])
docs_in_clusters = []
word_dictionaries = []
for c in range(clusters):
    df_cluster = cluster_labels_df[cluster_labels_df.Cluster == c]

    wordcloud_image, words_to_score = cluster_to_image(df_cluster)
    word_dictionaries.append(words_to_score)
    plt.imshow(wordcloud_image, interpolation="bilinear")
    plt.axis('off')
    plt.show()
    
    # Get the number of documents in each cluster
    # for analysis of how well distributed the clusters are
    docs = np.where(km.labels_ == c)[0]
    docs_in_clusters.append(len(docs))
def plot_word_dict(word_dict):
    """
    Plots frequency distribution of word-to-TFIDF score dictionaries.
    """
    plt.figure()
    x = range(len(word_dict.values()))
    plt.plot(x, list(word_dict.values()))
    plt.xticks(x, word_dict.keys(), rotation=90)
    plt.ylabel('summed TFIDF value')
for c in range(len(word_dictionaries)):
    plot_word_dict(word_dictionaries[c])
    plt.title('cluster {}'.format(c + 1))
_ = plt.bar(range(1, clusters + 1), docs_in_clusters)
plt.xlabel('Clusters')
plt.ylabel('Num bullets (Requirements)')
# Plot all clusters and wordclouds together
km = KMeans(n_clusters=6, random_state=42)
km.fit(norm)
labels = km.labels_
df_cluster_labels = pd.DataFrame({'Index': range(labels.size), 'Cluster': labels})

rows, cols = 3, 2
figure, axes = plt.subplots(rows, cols, figsize=(20, 15))
cluster_groups = list(df_cluster_labels.groupby('Cluster'))
cluster_counter = 0
for r in range(rows):
    for c in range(cols):
        cluster_counter += 1
        _, df_cluster = cluster_groups.pop(0)
        wordcloud_image, words_to_score = cluster_to_image(df_cluster)
        ax = axes[r][c]
        ax.imshow(wordcloud_image, interpolation="bilinear")
        ax.set_xticks([])
        ax.set_yticks([])
        ax.set_title('cluster {}'.format(cluster_counter))

plt.show()
# Compare with a few job descriptions
bullet_point_df = pd.DataFrame({'bullet_point': bullet_points}, index=range(len(bullet_points)))
# set seed for reproducibility in random sampling
np.random.seed(42)

for c in range(6):
    cluster_indices = cluster_labels_df[cluster_labels_df['Cluster'] == c].index
    cluster_bullet_points = bullet_point_df.loc[cluster_indices, 'bullet_point'].values
    try:
        random_bullets = np.random.choice(cluster_bullet_points, 3)
        print('3 random bullet points from cluster {}:'.format(c + 1))
        for r in random_bullets:
            print('-', r)
            print('\n')
    except: # when the bullet is empty
        pass
# Save files for next step
cluster_labels_df.to_pickle('cluster_labels_df.pk')
bullet_point_df.to_pickle('bullet_point_df.pk')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10308920513710518, 'fraction_numerical': 0.012495661228740022, 'mean_word_length': 4.405878674171357, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '241107', 'license': 'MIT', 'n_tokens_mistral': 2950, 'n_tokens_neox': 2818, 'n_words': 720, 'path': 'Resume_job_postings_part3.ipynb', 'repo_name': 'sbuitragodo/classifying-resumes-nlp-project', 'size': '8643'}