<jupyter_start><jupyter_text>Plotting  Chapters:* [How to plot with Pandas and Bokeh](How-to-plot-with-Pandas-and-Bokeh) Our data set is already prepared in a csv format.<jupyter_code># import what we need
import os
import numpy as np
import pandas as pd
import bokeh
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from bokeh.plotting import figure, show, output_notebook
from bokeh.models import HoverTool, ColumnDataSource, value
from bokeh.palettes import Colorblind, inferno
import spacy
nlp = spacy.load('en')
matplotlib.style.use('ggplot')
%matplotlib inline  
output_notebook()<jupyter_output><empty_output><jupyter_text>Reload the data into Pandas.<jupyter_code>BASE = os.path.dirname(os.path.dirname(os.path.abspath(__name__)))
DF_PARQUET = os.path.join(BASE, "data/cleaned/offers.parquet")
df = pd.read_parquet(DF_PARQUET, engine="pyarrow")<jupyter_output><empty_output><jupyter_text>How to plot with Pandas and Bokeh We count how many times each tag is used:<jupyter_code>tags_counting = df['tag'].value_counts()
tags_counting.head(20)
len(tags_counting)
tags_counting.describe()<jupyter_output><empty_output><jupyter_text>Number of tags used less than 10 times<jupyter_code>less_common_tags = tags_counting[tags_counting < 10]
len(less_common_tags)
most_used_tags = tags_counting[:12]
ax = most_used_tags.plot(
    kind='bar',
    figsize=(10,5),
    fontsize=15,
    title="12 Most used tags"
)
ax.axhline(y=tags_counting.mean(), color='b', label='Average', linestyle='--', linewidth=1)
ax.set(xlabel='Tags', ylabel='Job offers with this tag')
plt.show()
other_tags = tags_counting[-40:]
other_tags<jupyter_output><empty_output><jupyter_text>Now want to consider the each job offer, without considering tags, so they must be unique.<jupyter_code>grouped = df.drop(["tag"], axis=1)
grouped = grouped.drop_duplicates()
grouped.head()
sorted_number_offers = grouped.groupby("company")['job_id'].count().sort_values(ascending=False)
sorted_number_offers.describe()
# first 10 companies with most job offers
ax = sorted_number_offers[:10].plot(
    title="Job offers by companies (first 10 by number of offers)",
    kind='bar',
    figsize=(20,10),
    fontsize=15,
    legend=False,
)
ax.axhline(y=sorted_number_offers.mean(), color='b', label='Average', linestyle='--', linewidth=1)
ax.set(xlabel='Companies', ylabel='Number of job offers')
ax.legend(*ax.get_legend_handles_labels())
plt.show()
# last 10 companies per number of job offers
ax = sorted_number_offers[-10:].plot(
    kind='bar',
    figsize=(10,10),
    title="Job offers by companies (last 10 by number of offers)",
    fontsize=15
)
ax.set(xlabel='Companies', ylabel='Number of job offers')
plt.show()
grouped['weekday'] = grouped['published_on'].dt.dayofweek
grouped.head()
grouped_by_day = grouped.groupby("weekday")['job_id'].count()
grouped_by_day
grouped_by_day.index = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
ax = grouped_by_day.plot(
    kind='bar',
    figsize=(15,5),
    title="Weekly trend of new offers",
    fontsize=15)
ax.set(xlabel='Days', ylabel='Number of published job offers')
plt.show()
dt_df = grouped.set_index("published_on")
dt_df = dt_df.loc[:,["job_id"]].resample('D', label="published_on").count()
dt_df.describe()
fig, ax = plt.subplots(figsize=(20, 10), )
ax.bar(dt_df.index, dt_df['job_id'], align='center')
ax.axhline(y=dt_df['job_id'].mean(), color='b', label='Average', linestyle='--', linewidth=1)
ax.set(xlabel='Day', ylabel='New job offers published')
ax.legend(*ax.get_legend_handles_labels())
ax.xaxis.set_minor_locator(matplotlib.dates.WeekdayLocator(byweekday=(1),interval=1))
ax.xaxis.set_minor_formatter(matplotlib.dates.DateFormatter('%d-%m-%y'))
ax.xaxis.grid(True, which="minor")
ax.xaxis.grid(False, which="major")
ax.set_title('Number of job offers published on each day')
plt.gcf().autofmt_xdate()
plt.setp(ax.get_xmajorticklabels(), visible=False)
plt.show()
month_df = grouped.set_index("published_on").loc[:,["job_id"]]
month_df = month_df.resample('MS', label="published_on").count()
month_offers = pd.Series(month_df['job_id'].values, index=['September', 'October', 'November'])
ax = month_offers.plot(
    kind='bar',
    figsize=(20, 10),
    title="Monthly trend for new job offers",
    fontsize=15,
)
ax.set(xlabel='Month', ylabel='Number of new job offers published')
plt.show()<jupyter_output><empty_output><jupyter_text>Exercise: Can you find the most Pythonic companies?<jupyter_code>python_offers = df.loc[df.description.str.contains("python", case=False) ]
pythonic_companies = python_offers.groupby("company")[["job_id"]].count()
pythonic_companies = pythonic_companies[pythonic_companies["job_id"] >10].sort_values(by="job_id", ascending=False)
ax = pythonic_companies.plot(
    kind="bar",
    figsize=(10,5),
    legend=False
)
ax.set_title('Job offers related to Python')
ax.set(xlabel='Company', ylabel='Number of job offers')
plt.show()
java_offers = df.loc[df.description.str.contains("java", case=False)]
java_companies = java_offers.groupby("company")[["job_id"]].count()
java_companies = java_companies[java_companies["job_id"] >15].sort_values(by="job_id", ascending=False)
ax = java_companies.plot(
    kind="bar",
    figsize=(10,5),
    legend=False
)
ax.set_title('Job offers related to Java*')
ax.set(xlabel='Company', ylabel='Number of job offers')
plt.show()
data_offers = grouped.loc[df.role.str.contains("data", case=False)]
data_companies = data_offers.groupby("company")[["job_id"]].count()
data_companies = data_companies[data_companies["job_id"] >=2].sort_values(by="job_id", ascending=False)
ax = data_companies.plot(
    kind="bar",
    figsize=(10,5),
    legend=False
)
ax.set_title('Job offers related to Data*')
ax.set(xlabel='Company', ylabel='Number of job offers')
plt.show()
len_descriptions = grouped['description'].apply(len)
len_descriptions[len_descriptions > 4500]
descriptions = grouped[['job_id', 'description']]
descriptions.head()
doc = nlp(descriptions.iloc[0].description)
token_text = [token.orth_ for token in doc]
token_pos = [token.pos_ for token in doc]
token_lemma = [token.lemma_ for token in doc]
token_shape = [token.shape_ for token in doc]

pd.DataFrame(list(zip(token_text, token_lemma, token_shape)),
             columns=['token_text', 'token_lemma', 'token_shape'])
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(decode_error='ignore', use_idf=True, max_df=0.5,
                        min_df=2, stop_words='english',
                       )
tfidfOut = tfidf.fit_transform(descriptions.description)
tfidfOut.shape
from sklearn.decomposition import TruncatedSVD
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (50,50)
lsa = TruncatedSVD(n_components=2)
lsaOut = lsa.fit_transform(tfidfOut.todense())
xs, ys = lsaOut[:,0], lsaOut[:,1]
km = KMeans(n_clusters=2, init='k-means++', max_iter=100, n_init=1)
km.fit(lsaOut)
d = pd.DataFrame(lsaOut, columns=['x', 'y'])
d['company'] = grouped['company']
d['description'] = grouped['description']
d['role'] = grouped['role']
d['label'] = km.labels_
colormap = inferno(d['label'].nunique())
colors = [colormap[x] for x in d['label'].values]
bokeh_dat = ColumnDataSource(data=dict(x=d.x.values,y=d.y.values,role=d.role.values, color=colors))
# Plotting with Bokeh
clustering_plot = figure(title='Clustering English vs German offers',
                   plot_width = 800,
                   plot_height = 800,
                   tools= (u'pan, wheel_zoom, box_zoom,'
                           u'box_select, reset'),
                   active_scroll=u'wheel_zoom')
clustering_plot.circle(x='x', y='y',source=bokeh_dat, color='color', size=8.5)
clustering_plot.add_tools( HoverTool(tooltips = u'@role') )
clustering_plot.title.text_font_size = value(u'16pt')
show(clustering_plot);
# 1 for English
# 0 for German
d['label'].value_counts()
d.loc[d['label']==0].head()
row_index = d['label'] == 0
english_offers = d.loc[row_index, :]
english_offers.is_copy = False
english_offers.shape
tfidf = TfidfVectorizer(decode_error='ignore', use_idf=True, max_df=0.5,
                        min_df=2, stop_words='english',
                       )
tfidfOut = tfidf.fit_transform(english_offers.description)
lsa = TruncatedSVD(n_components=2)
lsaOut = lsa.fit_transform(tfidfOut.todense())
n_clusters = 5
km = KMeans(n_clusters=n_clusters, init='k-means++', max_iter=100, n_init=1)
km.fit(lsaOut)
english_offers.loc[:,'label'] = km.labels_
english_offers.loc[:, 'x'] = lsaOut[:,0]
english_offers.loc[:, 'y'] = lsaOut[:,1]
english_offers.head()
colormap = inferno(english_offers['label'].nunique())
colors = [colormap[x] for x in english_offers['label'].values]
bokeh_dat = ColumnDataSource(
    data=dict(
        x=english_offers.x.values,
        y=english_offers.y.values,
        role=english_offers.role.values,
        color=colors
    )
)
# create the plot and configure the
# title, dimensions, and tools
title = f"Clusters of english job offers (n={n_clusters}) after LSA"
cluster_plot = figure(title=title,
                   plot_width = 900,
                   plot_height = 800,
                   tools= (u'pan, wheel_zoom, box_zoom,'
                           u'box_select, reset'),
                   active_scroll=u'wheel_zoom')

# draw the words as circles on the plot
cluster_plot.circle(x='x', y='y', source=bokeh_dat, color='color', size=8.5)

# add our DataFrame as a ColumnDataSource for Bokeh

# add a hover tool to display words on roll-over
cluster_plot.add_tools( HoverTool(tooltips = u'@role') )

# configure visual elements of the plot
cluster_plot.title.text_font_size = value(u'16pt')

show(cluster_plot);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1492522024175374, 'fraction_numerical': 0.013419381274329031, 'mean_word_length': 5.556749496306246, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '638877', 'license': 'CC0-1.0', 'n_tokens_mistral': 3612, 'n_tokens_neox': 3457, 'n_words': 675, 'path': 'notebooks/chapter_2.ipynb', 'repo_name': 'pandatje/PyLadies_BerlinJobs', 'size': '9762'}