<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Importing libraries2&nbsp;&nbsp;Function definitions3&nbsp;&nbsp;Collecting data science job ids from Glassdoor4&nbsp;&nbsp;Reading Indeed data5&nbsp;&nbsp;Reading old data from previous project6&nbsp;&nbsp;Gathering atmost 2-3 years old data from other sources7&nbsp;&nbsp;Merging all data science jobs data8&nbsp;&nbsp;Getting BERT Sentence embeddings and adding them to the database9&nbsp;&nbsp;Saving jobs as MongoDB database  Importing libraries<jupyter_code>from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException
from selenium import webdriver
import time
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from fake_useragent import UserAgent
from urllib.request import urlopen, Request
from urllib.parse import urlparse
from bs4 import BeautifulSoup as soup

from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("<style>.output_result { max-width:100% !important; }</style>"))

#display(HTML('<style>.prompt{width: 0px; min-width: 0px; visibility: collapse}</style>'))

import warnings
warnings.filterwarnings('ignore')
warnings.filterwarnings("ignore", category=UserWarning, module='bs4')

pd.options.display.max_rows = 999<jupyter_output><empty_output><jupyter_text>Function definitions<jupyter_code>#Initializes driver
def initialize_driver(path):
    #Initializing the webdriver
    options = webdriver.ChromeOptions()
    
    ua = UserAgent()
    userAgent = ua.random
    options.add_argument(f'user-agent={userAgent}')
    
    #makes the driver more 'undetectale'
    options.add_argument("--disable-blink-features"), options.add_argument("--disable-blink-features=AutomationControlled")
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('useAutomationExtension', False)
    
    options.add_argument('--disable-extensions')
    options.add_argument('--profile-directory=Default')
    options.add_argument("--incognito")
    options.add_argument("--disable-plugins-discovery")
    options.add_argument("--start-maximized")
    
    #Change the path to where chromedriver is in your home folder.
    driver = webdriver.Chrome(executable_path=path, chrome_options=options)
    driver.maximize_window()

    return driver

#Brings job ids from job position and location search
def job_ids(driver, keyword, location,pagelimit=33):
    eids = []
    jids = []
    pagenum_limit = 33
    pagenum = 0
    
    try:
        # fill in search bar keyword field
        position_field = driver.find_element_by_xpath("//*[@id='sc.<EMAIL>']")
        position_field.send_keys(Keys.COMMAND+'a')
        position_field.send_keys(Keys.DELETE)
        position_field.send_keys(keyword)
        # fill in search bar location field
        location_field = driver.find_element_by_xpath("//*[@id='sc.<EMAIL>']")
        location_field.send_keys(Keys.COMMAND+'a')
        location_field.send_keys(Keys.DELETE)
        location_field.send_keys(location)

        # click search button and then view all jobs
        time.sleep(2)
        driver.find_element_by_xpath("//*[@type='submit']").click()

        time.sleep(3)
        driver.find_element_by_xpath("//a[@data-test='jobs-location-see-all-link']").click()

               
        while int(pagenum) <= pagelimit:
            
            
            # close a random popup if it shows up
            try:
                driver.find_element_by_xpath("//*[@id='JAModal']/div/div[2]/span").click()
            except NoSuchElementException:
                pass
            
            try:
                driver.find_element_by_css_selector('[alt="Close"]').click() #clicking to the X.
            except Exception:
                pass

            time.sleep(2)

            eids += [x.get_attribute('data-brandviews').split(':')[2][4:] for x in driver.find_elements_by_css_selector("li.react-job-listing")]
            jids += [x.get_attribute('data-id') for x in driver.find_elements_by_css_selector("li.react-job-listing")]


            # close a random popup if it shows up
            try:
                driver.find_element_by_xpath("//*[@id='JAModal']/div/div[2]/span").click()
            except NoSuchElementException:
                pass
            
            pages = driver.find_element_by_xpath("//div[@data-test='ResultsFooter']").text.split()
            pagenum = pages[1]
            if(pagenum == pages[3]):
                break
                
            # go to next page
            try:
                driver.find_element_by_xpath('.//a[@data-test="pagination-next"]').click()

            except NoSuchElementException:
                pass
        return eids, jids
        
        
    except NoSuchElementException:
        pass
def parse_url(url):
    try:
        # define headers to be provided for request authentication
        headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) ' 
                        'AppleWebKit/537.11 (KHTML, like Gecko) '
                        'Chrome/23.0.1271.64 Safari/537.11',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Accept-Encoding': 'none',
            'Accept-Language': 'en-US,en;q=0.8',
            'Connection': 'keep-alive'}
        request_obj = Request(url = url, headers = headers)
        opened_url = urlopen(request_obj)
        page_html = opened_url.read()
        opened_url.close()
        page_soup = soup(page_html, "html.parser")
                
        if type(page_soup.find("div", class_="css-ur1szg e11nt52q0").find("span", class_="css-1pmc6te e11nt52q4")) == type(None):
            rating=''
            company=page_soup.find("div", class_="css-16nw49e e11nt52q1").getText()

        else:
            rating = page_soup.find("div", class_="css-ur1szg e11nt52q0").find("span", class_="css-1pmc6te e11nt52q4").getText()
            company = page_soup.find("div", class_="css-16nw49e e11nt52q1").getText().replace(rating,'')
            rating = rating[:-1]

        position = page_soup.find("div", class_="css-17x2pwl e11nt52q6").getText()
        jobloc = page_soup.find("div", class_="css-1v5elnn e11nt52q2").getText()
        
        if type(page_soup.find("span", class_="small css-10zcshf e1v3ed7e1")) == type(None):
            salary=''
        else:
            salary = page_soup.find("span", class_="small css-10zcshf e1v3ed7e1").getText()#.split(':')[1].strip()
        desc = ' '.join([para.get_text() for para in page_soup.find("div", {'id':"JobDescriptionContainer"}).find_all(['li','p'])])
        
    
    except Exception as e:
        print(e)
        rating=''
        company=''
        position=''
        jobloc=''
        salary=''
        desc=''
    return position, desc, salary, jobloc, company, rating
def clean_text(s):
    # Fixing Unicode encoding errors
    s=ftfy.fix_text(s)
    # Fixing phds
    s=s.replace('PhD','phd')
    s=s.replace('Ph.D','phd')
    # Fixing camel cases
    s=re.sub(r'((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))', r' \1', s)
    # Removing non-ASCII characters
    s=''.join(i for i in s if ord(i)<128)
    # Lowercasing
    s=s.lower()
    # Replacing phd, ms,bs,eg
    s=s.replace('ph.d.','phd')
    s=s.replace('ph,d.','phd')
    s=s.replace('p.h.d.','phd')
    s=s.replace('m.sc', 'msc')
    s=s.replace('m.s.', 'msc')
    s=s.replace("master's", 'msc')
    s=s.replace("bachelor's",'bsc')
    s = re.sub(r"e\.g\.", '', s)
    s = re.sub(r"it’s", 'it is', s)
    s = re.sub(r"we’re", 'we are', s)
    # Removing [ and ]
    s=s.replace('\\[', '')
    s=s.replace('\\]', '')
    # Removing punctuations
    s=s.replace('\\|', ' ')
    # ()<>&/ to comma and space
    s = re.sub(r"[()<>/]", ', ', s)
    # ()<>&/ to comma and space
    s = re.sub(r"&", 'and', s) 
    # ?! to dot and space
    s = re.sub(r"[?!]", '. ', s) 
    # email address to dot
    s = re.sub(r" [a-z0-9]+[\.'\-a-z0-9_]*[a-z0-9]+@\w+\.com", "", s) 
    # Removing #
    s = re.sub(r'[#"\']', '', s)
    # Removing \n and \r
    s = re.sub(r"[\t\n\r\f\v]+", ". ", s) 
    # Adding space after comma and dot
    s = re.sub(r'(?<=[.,])(?=[^\s])', r' ', s) 
    # Removing the empty space before a dot
    s = re.sub(r'\W+\\.', '.', s) 
    # Removing the empty space before a comma
    s = re.sub(r'\W+\,', ',', s) 
    # multiple dots to one dot
    s = re.sub(r'[,\.]+\\.+', '.', s) 
    # Replacing multiple whitespace by one whitespace
    s = re.sub(r' +',' ',s) 
    
    # Removing punctuation
    #punctuations=string.punctuation.replace("'","")+'—'+'–'
    #s=s.translate(str.maketrans(punctuations, ' '*len(punctuations)))
    
    # Removing hyperlinks
    s=re.sub(r'http\S+', ' ', s)
    # Removing extra spaces, tabs, and line breaks
    #s=' '.join(s.split())

    return s

from bertify import BERTify

en_bertify = BERTify(
    lang="en",
    last_four_layers_embedding=False
)<jupyter_output><empty_output><jupyter_text>Collecting data science job ids from Glassdoor<jupyter_code>driver = initialize_driver("/usr/local/bin/chromedriver")
url = "https://www.glassdoor.com/blog/tag/job-search/"
driver.get(url)

locations=["Alabama", "Alaska", "Arizona", "Arkansas", "Colorado", "Connecticut", "Delaware", "Florida", 
             "Georgia", "Hawaii", "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana",
             "Maine", "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", 
             "Montana", "Nebraska", "Nevada", "New Hampshire", "New Jersey", "New Mexico", "North Carolina",
             "North Dakota", "Ohio", "Oklahoma", "Oregon", "Pennsylvania", "Rhode Island", "South Carolina",
             "South Dakota", "Tennessee", "Texas", "Utah", "Vermont", "Virginia", "Washington State", 
             "West Virginia", "Wisconsin", "Wyoming", "Washington, DC", "Los Angeles, ca", "San Diego, ca",
             "San Jose, ca", "San Francisco, ca", "Fresno, ca", "Sacramento, ca", "Long Beach, ca", 
             "Oakland, ca", "Bakersfield, ca", "Anaheim, ca", "Stockton, ca", "Riverside, ca", 
             "santa ana, ca", "Irvine, ca", "Chula Vista, ca", "Fremont, ca", "Santa Clarita, ca", 
             "San Bernardino, ca", "Modesto, ca", "Moreno Valley, ca", "New York, NY"]
keyword='data scientist'

for location in locations:
    print('Getting Job IDs for ', location)
    eids, jids = job_ids(driver,'data scientist', location, pagelimit=33)
    time.sleep(1)
    pd.DataFrame({'eids':eids, 'jids':jids}).to_csv("_".join(keyword.split()+location.split(', '))+'.csv')
  
#for loc in ${locations[@]}; do python glassdoor_scraper.py "data scientist" "$loc"; echo $loc; done
locations=["Alabama", "Alaska", "Arizona", "Arkansas", "Colorado", "Connecticut", "Delaware", "Florida", 
             "Georgia", "Hawaii", "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana",
             "Maine", "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", 
             "Montana", "Nebraska", "Nevada", "New Hampshire", "New Jersey", "New Mexico", "North Carolina",
             "North Dakota", "Ohio", "Oklahoma", "Oregon", "Pennsylvania", "Rhode Island", "South Carolina",
             "South Dakota", "Tennessee", "Texas", "Utah", "Vermont", "Virginia", "Washington State", 
             "West Virginia", "Wisconsin", "Wyoming", "washington, dc", "Los Angeles, ca", "San Diego, ca",
             "San Jose, ca", "San Francisco, ca", "Fresno, ca", "Sacramento, ca", "Long Beach, ca", 
             "Oakland, ca", "Bakersfield, ca", "Anaheim, ca", "Stockton, ca", "Riverside, ca", 
             "santa ana, ca", "Irvine, ca", "Chula Vista, ca", "Fremont, ca", "Santa Clarita, ca", 
             "San Bernardino, ca", "Modesto, ca", "Moreno Valley, ca", "New York, NY"]
keyword='data scientist'

df = pd.read_csv("~/Desktop/Metis/DE_project/"+keyword+'-'+'-'.join(locations[0].split(', '))+'-job-results.csv')

for i in range(1, len(locations)):
    df2 = pd.read_csv("~/Desktop/Metis/DE_project/"+keyword+'-'+'_'.join(locations[i].split(', '))+'-job-results.csv')
    df = pd.concat([df, df2])


df[~df.jobid.duplicated()]<jupyter_output><empty_output><jupyter_text>Reading Indeed data<jupyter_code>indeed=pd.read_pickle('indeedjobs_new_df.pickle')

indeed['salary']=''
indeed['location']=indeed['city']
indeed=indeed[['location','title','description','salary','city','company','company_rating','url']]
indeed.rename(columns={'title':'position','description':'desc','city':'jobloc','company_rating':'rating',
                      'url':'jobid'}, inplace=True)

indeed.head()
all_jobs = pd.concat([df, indeed]).reset_index(drop=True)
all_jobs.head()
all_jobs.shape<jupyter_output><empty_output><jupyter_text>Reading old data from previous project<jupyter_code># reading old Indeed data from previous project
in_old = pd.read_pickle('indeed_jobs.pickle')
in_old

in_old['salary']=''
in_old['location']=in_old['city']
in_old=in_old[['location','title','description','salary','city','company','company_rating','url']]
in_old.rename(columns={'title':'position','description':'desc','city':'jobloc','company_rating':'rating',
                      'url':'jobid'}, inplace=True)
in_old
#old Glassdoor data from previous project
gl_old = pd.read_csv('glassdoor_data_science_jobs.csv').iloc[:, 1:]
gl_old['location']=gl_old['jobloc']
gl_old['jobid']=''


gl_old=gl_old[['location','position','desc','salary','jobloc','company','rating','jobid']]
gl_old
old_jobs= pd.concat([gl_old, in_old]).reset_index(drop=True)
old_jobs.head()<jupyter_output><empty_output><jupyter_text>Gathering atmost 2-3 years old data from other sources<jupyter_code>#github repo https://github.com/yanmsong/Skills-Extraction-from-Data-Science-Job-Postings/tree/main/data
gl_da=pd.read_csv('Skills-Extraction-from-Data-Science-Job-Postings-main/data/glassdoor_data_analyst.csv').iloc[:, 1:]
gl_ds=pd.read_csv('Skills-Extraction-from-Data-Science-Job-Postings-main/data/glassdoor_data_scientist.csv').iloc[:, 1:]
in_da=pd.read_csv('Skills-Extraction-from-Data-Science-Job-Postings-main/data/indeed-da-0407.csv').iloc[:, 1:]
in_ds=pd.read_csv('Skills-Extraction-from-Data-Science-Job-Postings-main/data/indeed-ds-0407.csv').iloc[:, 1:]
li_da=pd.read_csv('Skills-Extraction-from-Data-Science-Job-Postings-main/data/linkedin_data_analyst.csv')#.iloc[:, 1:]
li_ds=pd.read_csv('Skills-Extraction-from-Data-Science-Job-Postings-main/data/linkedin_data_scientist.csv')#.iloc[:, 1:]

li_da['rating']=''
li_da = li_da[['City', 'Name','Description', 'Salary','Location','Company','rating','Url']]


li_da.rename(columns={'City':'location', 'Name':'position','Description':'desc','Salary':'salary',
                      'Company':'company', 'Location':'jobloc','Url':'jobid'}, inplace=True)

li_ds['rating']=''
li_ds = li_ds[['City', 'Name','Description', 'Salary','Location','Company','rating','Url']]


li_ds.rename(columns={'City':'location', 'Name':'position','Description':'desc','Salary':'salary',
                      'Company':'company', 'Location':'jobloc','Url':'jobid'}, inplace=True)

gl_ds['rating']=''
gl_ds = gl_ds[['City', 'Name','Description', 'Salary','Location','Company','rating','Url']]


gl_ds.rename(columns={'City':'location', 'Name':'position','Description':'desc','Salary':'salary',
                      'Company':'company', 'Location':'jobloc','Url':'jobid'}, inplace=True)

gl_da['rating']=''
gl_da = gl_da[['City', 'Name','Description', 'Salary','Location','Company','rating','Url']]


gl_da.rename(columns={'City':'location', 'Name':'position','Description':'desc','Salary':'salary',
                      'Company':'company', 'Location':'jobloc','Url':'jobid'}, inplace=True)

in_ds['rating']=''
in_ds['jobid']=''
in_ds['position']='Data Scientist'
in_ds['location']=in_ds['Location']
in_ds = in_ds[['location', 'position','Description', 'Salary','Location','Company','rating','jobid']]


in_ds.rename(columns={'Description':'desc','Salary':'salary',
                      'Company':'company', 'Location':'jobloc'}, inplace=True)

in_da['rating']=''
in_da['jobid']=''
in_da['position']='Data Analyst'
in_da['location']=in_da['Location']
in_da = in_da[['location', 'position','Description', 'Salary','Location','Company','rating','jobid']]


in_da.rename(columns={'Description':'desc','Salary':'salary',
                      'Company':'company', 'Location':'jobloc'}, inplace=True)

github_job1 = pd.concat([gl_da,gl_ds,in_da,in_ds,li_da,li_ds]).reset_index(drop=True)
#github jobs from https://github.com/kneehit/Indeed-Job-Scraper
ds=pd.read_csv('Data Scientist.csv').iloc[:,1:]

ds['location']=ds['Location']
ds = ds[['location', 'Job_Title','Job_Description', 'Estimated_Salary',
         'Location','Company','Rating_In_Stars','Link_Job_Company']]


ds.rename(columns={'Job_Title':'position', 'Job_Description':'desc','Estimated_Salary':'salary',
                      'Company':'company', 'Location':'jobloc',
                  'Rating_In_Stars':'rating', 'Link_Job_Company':'jobid'}, inplace=True)

ai=pd.read_csv('Artificial Intelligence.csv').iloc[:,1:]

ai['location']=ai['Location']
ai = ai[['location', 'Job_Title','Job_Description', 'Estimated_Salary',
         'Location','Company','Rating_In_Stars','Link_Job_Company']]


ai.rename(columns={'Job_Title':'position', 'Job_Description':'desc','Estimated_Salary':'salary',
                      'Company':'company', 'Location':'jobloc',
                  'Rating_In_Stars':'rating', 'Link_Job_Company':'jobid'}, inplace=True)

ml=pd.read_csv('Machine Learning.csv').iloc[:,1:]

ml['location']=ml['Location']
ml = ml[['location', 'Job_Title','Job_Description', 'Estimated_Salary',
         'Location','Company','Rating_In_Stars','Link_Job_Company']]


ml.rename(columns={'Job_Title':'position', 'Job_Description':'desc','Estimated_Salary':'salary',
                      'Company':'company', 'Location':'jobloc',
                  'Rating_In_Stars':'rating', 'Link_Job_Company':'jobid'}, inplace=True)

github_job2 = pd.concat([ai,ml,ds]).reset_index(drop=True)
#github repo https://github.com/picklesueat/data_jobs_data
ds2 = pd.read_csv('DataScientist.csv').iloc[:,2:]
de2 = pd.read_csv('DataEngineer.csv')
da2 = pd.read_csv('DataAnalyst.csv')
ba2 = pd.read_csv('BusinessAnalyst.csv').iloc[:,2:]

ds2['location']=ds2['Location']
ds2['jobid']=''
ds2 = ds2[['location', 'Job Title','Job Description', 'Salary Estimate',
         'Location','Company Name','Rating','jobid']]


ds2.rename(columns={'Job Title':'position', 'Job Description':'desc','Salary Estimate':'salary',
                      'Company Name':'company', 'Location':'jobloc',
                  'Rating':'rating'}, inplace=True)

de2['location']=de2['Location']
de2['jobid']=''
de2 = de2[['location', 'Job Title','Job Description', 'Salary Estimate',
         'Location','Company Name','Rating','jobid']]


de2.rename(columns={'Job Title':'position', 'Job Description':'desc','Salary Estimate':'salary',
                      'Company Name':'company', 'Location':'jobloc',
                  'Rating':'rating'}, inplace=True)
da2['location']=da2['Location']
da2['jobid']=''
da2 = da2[['location', 'Job Title','Job Description', 'Salary Estimate',
         'Location','Company Name','Rating','jobid']]


da2.rename(columns={'Job Title':'position', 'Job Description':'desc','Salary Estimate':'salary',
                      'Company Name':'company', 'Location':'jobloc',
                  'Rating':'rating'}, inplace=True)
ba2['location']=ba2['Location']
ba2['jobid']=''
ba2 = ba2[['location', 'Job Title','Job Description', 'Salary Estimate',
         'Location','Company Name','Rating','jobid']]


ba2.rename(columns={'Job Title':'position', 'Job Description':'desc','Salary Estimate':'salary',
                      'Company Name':'company', 'Location':'jobloc',
                  'Rating':'rating'}, inplace=True)
github_job3 = pd.concat([de2, da2,ba2,ds2]).reset_index(drop=True)
#kaggle jobs
kaggle = pd.read_csv('glassdoor_kaggle_visible.csv')


kaggle['salary']=''
kaggle['jobid']=''
kaggle['location']=kaggle['gaTrackerData.location']
kaggle = kaggle[['location', 'gaTrackerData.jobTitle','job.description', 'salary',
         'gaTrackerData.location','header.employerName','header.rating','jobid']]

kaggle.rename(columns={'gaTrackerData.jobTitle':'position', 
                       'job.description':'desc', 
                       'header.employerName':'company', 
                       'gaTrackerData.location':'jobloc',
                       'header.rating':'rating'}, inplace=True)


kaggle.head()<jupyter_output><empty_output><jupyter_text>Merging all data science jobs data<jupyter_code>all_jobs = pd.concat([all_jobs, old_jobs, github_job1, github_job2, github_job3, kaggle]).reset_index(drop=True)
all_jobs.head()
all_jobs.to_csv('dsjobs.csv', index=False)
## Cleaning jobs
all_jobs.position = all_jobs.position.astype(str)
all_jobs.desc = all_jobs.desc.astype(str)
all_jobs.company = all_jobs.company.astype(str)
#drop records with null job descriptions
all_jobs=all_jobs[~all_jobs.desc.isnull()].reset_index(drop=True)
#remove duplicated job descriptions
all_jobs=all_jobs[~all_jobs.desc.duplicated()].reset_index(drop=True)
#filter short descriptions
all_jobs=all_jobs[all_jobs.desc.str.len()>150]#.description.loc[6143]
#cleaning job descriptions
all_jobs['cleaned_desc']=all_jobs.desc.apply(lambda s: clean_text(s))<jupyter_output><empty_output><jupyter_text>Getting BERT Sentence embeddings and adding them to the database<jupyter_code>job_en_embeddings = en_bertify.embedding(all_jobs.cleaned_desc.tolist()) 

all_jobs['sent_emb'] = pd.Series(list(job_en_embeddings))
all_jobs.reset_index(level=0, inplace=True)

jobs_json = all_jobs[~all_jobs.sent_emb.isnull()].to_json(orient='records')
with open('/content/drive/My Drive/jobs_json.json', 'w') as f:
    f.write(jobs_json)

# This json is used to create MongoDB database in MongoDB cluster<jupyter_output><empty_output><jupyter_text>Saving jobs as MongoDB database<jupyter_code>#save into mongodb
import pandas as pd
import pymongo
from pymongo import MongoClient
myclient = MongoClient('localhost')
mydb = myclient["dsjobs"]
mycol = mydb["mycollection"]

merged_dict = all_jobs.to_dict() # Convert to dictionary

mycol.insert_many(all_jobs.apply(lambda x: x.to_dict(), axis=1).to_list()) 

len(list(mycol.find()))
  
myclient.database_names()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18204771025596378, 'fraction_numerical': 0.010447501305937664, 'mean_word_length': 4.13822411093715, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 49, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '184605', 'license': 'MIT', 'n_tokens_mistral': 8112, 'n_tokens_neox': 7634, 'n_words': 1320, 'path': 'notebooks/1.0-MS-MergingCleaningSavingJobsMongoDB.ipynb', 'repo_name': 'mervesa/job_recommender', 'size': '22973'}