<jupyter_start><jupyter_text><jupyter_code>#Libraries
#Helpers

#EDA initial
#Preprocessing
#Eda Final
#Second Preprocessing

#Preprocessing Pipeline
#Model Core Pipeline
#Model Output Pipeline

#Testing<jupyter_output><empty_output><jupyter_text>Helpers<jupyter_code>def preprocessor(text):
    text = text.replace('\\r', '').replace('&nbsp', '').replace('\n', '')
    text = re.sub('<[^>]*>', '', text)
    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)', text)
    text = re.sub('[\W]+', ' ', text.lower()) +\
        ' '.join(emoticons).replace('-', '')
    return text

def coordinates(zipcode):
    zipcode = search.by_zipcode(zipcode)
    community = zipcode.post_office_city
    return community, "{},{}".format(zipcode.lat, zipcode.lng)

def creating_coordinates(df, column):
    df["Community"] = ""
    df["Coordinates"] = "None, None"
    for zipcode in df[column].unique():
        community, coordinate = coordinates(zipcode)
        df.loc[df[column] == zipcode, "Coordinates"] =  str(coordinate)
        df.loc[df[column] == zipcode, "Community"] =  community
    return df
def user_exist(user):
    if len(users.loc[users['UserID'] == user]) == 0:
        return False
    return True

def has_coordinates(user):
    c = users.loc[users['UserID'] == user, "Coordinates"] 

    if (len(c) == 0) or (c.iloc[0].split(',')[0]) == 'None':
        return False
    return True
    

def user_profile(user_id):
    user_profile = np.array(users.loc[users['UserID'] == user_id, ['DegreeType', 'Major']])[0]
    historical_apps = user_history.loc[user_history.UserID == user_id, 'JobTitle']
    user_profile = """
    Degree Type: {}
    Major: {}
    """.format(user_profile[0], user_profile[1])

    return user_profile

def historical_application(user_id):
    historical_apps = apps.loc[apps.UserID == user_id]['JobID']
    content = list()
    for application in historical_apps:
        temp = jobs.loc[jobs.JobID == application, ['Title', 'Description', 'Requirements']]
        if len(temp) != 0:
            content += [temp.Title.values + ". " + temp.Description.values + ". " + temp.Requirements.values]
    return content

class EpochLogger(CallbackAny2Vec):
    '''Callback to log information about training'''

    def __init__(self):
        self.epoch = 0

    def on_epoch_begin(self, model):
        print("Epoch #{} start".format(self.epoch))
        self.epoch += 1<jupyter_output><empty_output><jupyter_text>Libraries<jupyter_code>!pip install uszipcode
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
import ast 
from scipy import stats
from ast import literal_eval
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics.pairwise import linear_kernel, cosine_similarity
# from nltk.stem.snowball import SnowballStemmer
# from nltk.stem.wordnet import WordNetLemmatizer
# from nltk.corpus import wordnet
# from surprise import Reader, Dataset, SVD, evaluate
from gensim.models import Doc2Vec
import gensim.models.doc2vec
from collections import OrderedDict
import multiprocessing

from gensim.models.doc2vec import Doc2Vec, TaggedDocument
from gensim.utils import simple_preprocess
from gensim.test.utils import get_tmpfile
from gensim.models.callbacks import CallbackAny2Vec

import pickle

from uszipcode import SearchEngine
from  geopy.distance import distance

import gc
import re

import warnings; warnings.simplefilter('ignore')<jupyter_output><empty_output><jupyter_text>Checking Cores<jupyter_code>assert gensim.models.doc2vec.FAST_VERSION > -1, "SLOW VERSION"
MAX_DISTANCE = 15

#!dir .\Data\*.tsv<jupyter_output><empty_output><jupyter_text>Reading Data<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
folder = 'Data'

apps = pd.read_csv('./'+folder+'/apps.tsv', delimiter='\t',encoding='utf-8')
user_history = pd.read_csv('./'+folder+'/user_history.tsv', delimiter='\t',encoding='utf-8')
jobs = pd.read_csv('./'+folder+'/jobs.tsv', delimiter='\t',encoding='utf-8', error_bad_lines=False)
users = pd.read_csv('./'+folder+'/users.tsv' ,delimiter='\t',encoding='utf-8')
test_users = pd.read_csv('./'+folder+'/test_users.tsv', delimiter='\t',encoding='utf-8')<jupyter_output>b'Skipping line 122433: expected 11 fields, saw 12\n'
b'Skipping line 602576: expected 11 fields, saw 12\n'
b'Skipping line 990950: expected 11 fields, saw 12\n'<jupyter_text>EDA 1<jupyter_code>apps.head()

user_history.head()
jobs.head()
users.head()
test_users.head()
len(users)<jupyter_output><empty_output><jupyter_text>ETL Pipeline - Preprocessing  Subsetting & Sampling  NY<jupyter_code>users = users.loc[users.State == 'NY']
len(users)
users = users.sample(frac=0.2, replace=False, random_state=1)
len(users)
jobs = jobs.loc[jobs.State == 'NY']
all_jobs = jobs.copy()
#jobs.to_csv("NYjobs.tsv",  sep='\t',encoding='utf-8')<jupyter_output><empty_output><jupyter_text>Subsetting jobs with zipcode for testing distance feature<jupyter_code>jobs.Zip5 = jobs.Zip5.fillna(0)
jobs.Zip5 = jobs.Zip5.astype(int)
jobs = jobs.loc[jobs['Zip5'] != 0]
len(jobs)
jobs = jobs.sample(frac=0.2, replace=False, random_state=1)
len(jobs)<jupyter_output><empty_output><jupyter_text>Clearning Memory<jupyter_code>gc.collect()<jupyter_output><empty_output><jupyter_text>Creating jobs coordinates and Communities<jupyter_code>search = SearchEngine(simple_zipcode=True)

search.by_zipcode("2e").lat
%%timeit
jobs["Community"] = ""
jobs["Coordinates"] = ""
for zipcode in jobs.Zip5.unique():
    community, coordinate = coordinates(zipcode)
    jobs.loc[jobs.Zip5 == zipcode, "Coordinates"] =  str(coordinate)
    jobs.loc[jobs.Zip5 == zipcode, "Community"] =  community<jupyter_output>1 loop, best of 5: 1.56 s per loop<jupyter_text>Creating user coordinates<jupyter_code>users = creating_coordinates(users, 'ZipCode')<jupyter_output><empty_output><jupyter_text>Cleaning Description and Requirements<jupyter_code>jobs['Description'] = jobs['Description'].astype(dtype='str').apply(preprocessor)
jobs['Requirements'] = jobs['Requirements'].astype(dtype='str').apply(preprocessor)<jupyter_output><empty_output><jupyter_text>Making jobs profiles<jupyter_code>jobs['profile'] = jobs['Title'].astype(str)  +  '. ' + jobs['Requirements'].astype(str) +  '. ' + jobs['Description'].astype(str)<jupyter_output><empty_output><jupyter_text>Making users profiles<jupyter_code>%%timeit

users['profile'] =  (users['Major'].astype(str) +  " " ) + (users['DegreeType'].astype(str) + " ")

#Cleaning Garbage Words
users['profile'] = users['profile'].str.replace('None.', ' ')
users['profile'] = users['profile'].str.replace('Not Applicable', ' ')
users['profile'] = users['profile'].str.replace('nan', ' ')<jupyter_output>100 loops, best of 5: 13.6 ms per loop<jupyter_text>EDA 2  Jobs by Communmties/County<jupyter_code>jobs.groupby(['Community']).size().reset_index(name='Count').sort_values('Count', ascending=False).head(20)<jupyter_output><empty_output><jupyter_text>Jobs by zipcodes<jupyter_code>jobs.groupby(['Community', 'Zip5', 'Coordinates']).size().reset_index(name='Count').sort_values('Count', ascending=False).head(20)<jupyter_output><empty_output><jupyter_text>Users by Community<jupyter_code>users.groupby(['Community']).size().reset_index(name='Count').sort_values('Count', ascending=False).head(20)<jupyter_output><empty_output><jupyter_text>Machine Learninig Pipeline: NLP Model<jupyter_code>%%time
def similarities_nlp_model( model_name = "jobs_doc2vec_model",
                 mapping_name = "jobID_mapping.p", max_epochs = 100,
                 alpha = 0.025):
    
    document = list()
    jobID_mapping = dict()
        
    for i, token in enumerate(jobs['profile']):
        value = jobs.iloc[i]["JobID"]
        tokens = TaggedDocument(simple_preprocess(token), [i])
        document.append(tokens)
        jobID_mapping[i] = value

    epoch_logger = EpochLogger()
    model = Doc2Vec(size = 20, alpha=alpha, 
                    min_alpha=0.00025, min_count=2,
                    callbacks=[epoch_logger], dm =1, workers=8, window=2)
    
    model.build_vocab(document)
    print(model.corpus_count)
    for epoch in range(max_epochs):
        print('iteration {0}'.format(epoch))
        model.train(document, 
                    total_examples=model.corpus_count, 
                    epochs=model.iter)
        # decrease the learning rate
        model.alpha -= 0.0002
        # fix the learning rate, no decay
        model.min_alpha = model.alpha

    # create a dictionary
    pickle.dump(model, open(model_name, "wb")) 
    pickle.dump(jobID_mapping, open(mapping_name, "wb")) 

# -------------------------------------------------------------
# Load the dictionary back from the pickle file.
    return model, jobID_mapping

model, jobID_mapping = similarities_nlp_model()<jupyter_output>6736
iteration 0
Epoch #0 start
Epoch #1 start
Epoch #2 start
Epoch #3 start
Epoch #4 start
iteration 1
Epoch #5 start
Epoch #6 start
Epoch #7 start
Epoch #8 start
Epoch #9 start
iteration 2
Epoch #10 start
Epoch #11 start
Epoch #12 start
Epoch #13 start
Epoch #14 start
iteration 3
Epoch #15 start
Epoch #16 start
Epoch #17 start
Epoch #18 start
Epoch #19 start
iteration 4
Epoch #20 start
Epoch #21 start
Epoch #22 start
Epoch #23 start
Epoch #24 start
iteration 5
Epoch #25 start
Epoch #26 start
Epoch #27 start
Epoch #28 start
Epoch #29 start
iteration 6
Epoch #30 start
Epoch #31 start
Epoch #32 start
Epoch #33 start
Epoch #34 start
iteration 7
Epoch #35 start
Epoch #36 start
Epoch #37 start
Epoch #38 start
Epoch #39 start
iteration 8
Epoch #40 start
Epoch #41 start
Epoch #42 start
Epoch #43 start
Epoch #44 start
iteration 9
Epoch #45 start
Epoch #46 start
Epoch #47 start
Epoch #48 start
Epoch #49 start
iteration 10
Epoch #50 start
Epoch #51 start
Epoch #52 start
Epoch #53 start
Epoch #5[...]<jupyter_text>Recommender Pipeline  Recomender by popularity/Cold Star<jupyter_code>unique = 0.3 #update name
top = 20

def ranking_by_popularity(top = 100):
    
    popular_jobs = user_history.groupby(
    ['JobTitle']).size().reset_index(
    name='Count').sort_values('Count', ascending=False)
    ranking =  dict()
    top_i = 0
    
    while True:
        job_title = popular_jobs['JobTitle'].iloc[top_i]
        jobs_list = jobs.loc[jobs['Title'] == job_title, ['JobID']]['JobID'].unique().tolist()

        if len(jobs_list) > 1:
            ranking[job_title] = jobs_list

        if len(ranking) == top:
            break

        top_i +=1
    return ranking

ranking_popular = ranking_by_popularity(top = 50)

def recommender_popular_jobs(user_id,  top = 10):
    
    recommended_popular_jobs = dict()
    c1 = users.loc[users['UserID'] == user_id, 'Coordinates']
    count = 0
    if user_exist(user_id) and has_coordinates(user_id):
        for title, jobs_list in ranking_popular.items():
            
            distances = dict()
        
            for job in jobs_list:
                
                c2 = jobs.loc[jobs['JobID'] == job, 'Coordinates']
                if c2.iloc[0].split(',')[0] == 'None':
                    continue

                distances[job] = round(distance(c1, c2.iloc[0]).miles, 2)
            
            distances = sorted(distances.items(), key=lambda kv: kv[1])
            closest = distances[0]
            if closest[1] >= MAX_DISTANCE:
                continue
            recommended_popular_jobs[title] = (closest[0], closest[1])
            count += 1
            if count == top:
                break
            
        if len (recommended_popular_jobs) < top:
            count = len (recommended_popular_jobs) -1
            
            for title, jobs_list in ranking_popular.items():
                recommended_popular_jobs[title] = jobs_list[0]
                count += 1
                if count == top:
                    break

            
    else:
        for title, jobs_list in ranking_popular.items():
            recommended_popular_jobs[title] = jobs_list[0]
            count += 1
            if count == top:
                break
            
    
    
    return recommended_popular_jobs<jupyter_output><empty_output><jupyter_text>Content Based Recommender<jupyter_code>#If user has coordinates
#if user has info then similarties if not popularity ranker
jobID_mapping = pickle.load(open("jobID_mapping.p", "rb"))
model  = pickle.load(open("jobs_doc2vec_model", "rb"))

def content_distance_based_recommender(user_id, jobID_mapping = jobID_mapping, model =  model, top = 10):
    #As infer_vector produce stochastics result I made a for to save the best list
    user_profile = np.array(users.loc[users['UserID'] == user_id, 'profile'])[0]

    historical_apps = historical_application(user_id)
    for application in historical_apps:
        user_profile += ". " + (str(application[0]) + " ")
    user_profile = simple_preprocess(user_profile)
    
    best = 0
    tops =pd.DataFrame(index = range(top), columns = ['JobID', 'Title', 'Distance', 'Description', 'Requirements'])
    c1 = users.loc[users['UserID'] == user_id, 'Coordinates']
    job_distance_list = list()
    
    for i in range (1):
        inferred_vector = model.infer_vector(user_profile)
        sims = model.docvecs.most_similar([inferred_vector], topn=len(model.docvecs))
        sum_results = 0
        count = 0
        total_recom = 0
        job_distance_list = list()
        
        while True:
            job_id = jobID_mapping[sims[count][0]]
            c2 = jobs.loc[jobs['JobID'] == jobID_mapping[sims[count][0]], 'Coordinates']
            count +=1
            if len(c2) == 0:
                #print("Empty", sims[count][0])
                continue
            
            if c2.iloc[0].split(',')[0] == 'None':
                #print('None', sims[count][0])
                continue
                
            ## bug here
            job_distance = round(distance(c1, c2).miles, 2)
            
            if job_distance >= MAX_DISTANCE:
                #print('Distance', job_distance,  sims[count][0])
                continue

            sum_results+=sims[count][1]
            total_recom +=1
            job_distance_list.append(job_distance)
            if total_recom == top:
                break
        
        ##Best simulation
        if sum_results > best:
            best = sum_results
            best_sim = sims
            job_distance_list2 = job_distance_list
            
    for i in range(top):
        recomendation = jobID_mapping[best_sim[i][0]]
        tops.iloc[i]['JobID', 'Title',  'Description', 'Requirements'] = np.array(jobs.loc[jobs['JobID'] == recomendation][['JobID', 'Title', 
                                                               'Description', 'Requirements']])[0]
        tops.iloc[i]['Distance'] = job_distance_list2[i]
    
    return tops 
 
        
def hybrid_recommender(user_id, top=10):
    
    if has_coordinates(user_id) and len(historical_application(user_id)) > 0:
        print("Content")
        return content_distance_based_recommender(user_id, top = top)
    else:
        print("Cold Star")
        return recommender_popular_jobs(user_id,  top = top)<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code>users.head(5)
#Cold 153286, 944873
#Content 248008, 1184529
user_id = 1165147

print("USER PROFILE: ", user_profile(user_id))
print("HISTORICAL APPLICATION CONTENT:\n", historical_application(user_id))
hybrid_recommender(user_id, top = 10) 

user_id = 1165147

print("USER PROFILE: ", user_profile(user_id))
print("HISTORICAL APPLICATION CONTENT:\n\n", historical_application(user_id))
content_distance_based_recommender(user_id, top = 10)
user_id = 1165147

print("USER PROFILE: ", user_profile(user_id))
print("HISTORICAL APPLICATION CONTENT:\n\n", historical_application(user_id))
recommender_popular_jobs(user_id,  top = 10)<jupyter_output>USER PROFILE:  
    Degree Type: Bachelor's
    Major: BA/Criminal Justice
    
HISTORICAL APPLICATION CONTENT:

 []<jupyter_text>How many user with content?<jupyter_code>counter = 0
for u in users.UserID.unique():
    if len(historical_application(u)) > 0:
        counter += 1

print(counter)
counter/len(users.UserID.unique())<jupyter_output>1204
{'fraction_non_alphanumeric': 0.13354707441889144, 'fraction_numerical': 0.022566126148344533, 'mean_word_length': 3.5574599606631074, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 92, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '179759', 'license': 'MIT', 'n_tokens_mistral': 5673, 'n_tokens_neox': 5321, 'n_words': 1123, 'path': 'Jobs_Recommender.ipynb', 'repo_name': '01kartiknair/Job_Recommender_SIN', 'size': '16219'}