<jupyter_start><jupyter_text>Job TitlesGiven a job title like "junior data analyst" categorize it as one of "finance", "sales", or "technology".<jupyter_code>%matplotlib inline
%matplotlib notebook
import numpy as np
import seaborn as sns
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split, cross_val_score
sns.set()<jupyter_output><empty_output><jupyter_text>Loading and inspecting dataset<jupyter_code>df = pd.read_csv('data/test.csv').append(pd.read_csv('data/train.csv'))
df.head()
job_map = {job: i for job, i in zip(df.loc[:, 'job_category'].unique(), range(3))}
df['job_category'] = df['job_category'].replace(job_map)
job_map<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>from sklearn.preprocessing import OneHotEncoder
from sklearn.feature_extraction.text import CountVectorizer

class_encoder = OneHotEncoder(categories='auto')
text_vectorizer = CountVectorizer(stop_words='english')

X = text_vectorizer.fit_transform(df['job_title'].values).toarray()
y = class_encoder.fit_transform(df['job_category'].values.reshape(-1, 1)).toarray()
X[0], y[0]<jupyter_output><empty_output><jupyter_text>Building &amp; testing the model<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=10)

dt = DecisionTreeClassifier(max_depth=5)
dt.fit(X_train, y_train)

print('Accuracy on training set:', dt.score(X_train, y_train))
print('Accuracy on test set:', dt.score(X_test, y_test))<jupyter_output>Accuracy on training set: 1.0
Accuracy on test set: 1.0<jupyter_text>Testing<jupyter_code>job_map_inverse = {job_map[key]: key for key in job_map}

def predict(s):
    global dt, class_encoder, text_vectorizer, job_map_inverse
    ps = text_vectorizer.transform([s]).toarray()
    prediction = dt.predict(ps)
    predicted_class = class_encoder.inverse_transform(prediction)[0][0]
    return job_map_inverse[predicted_class]

to_test = [
    'data analyst',
    'sales manager',
    'underwriter'
]

for pred, x in zip(map(predict, to_test), to_test):
    print('"' + x + '" ->', '"' + pred + '"')<jupyter_output>"data analyst" -> "technology"
"sales manager" -> "sales"
"underwriter" -> "finance"
{'fraction_non_alphanumeric': 0.1289348857266063, 'fraction_numerical': 0.00646830530401035, 'mean_word_length': 7.2562277580071175, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '349737', 'license': 'MIT', 'n_tokens_mistral': 838, 'n_tokens_neox': 814, 'n_words': 170, 'path': 'Problems/Job_Titles/Job_Titles.ipynb', 'repo_name': 'MartinKondor/MachineLearning', 'size': '2319'}