<jupyter_start><jupyter_text>Testing w/ 2 radiologists<jupyter_code>radiologists = ["<NAME>", "<NAME>"]
prefix = 'https://www.doximity.com/pub/'
user_agent = {'User-agent' : 'Mozilla/5.0'}
input_name = radiologists[0].replace(' ', '-').lower()
degree = "-md"
url = prefix+input_name+degree
page = requests.get(url, headers=user_agent)
soup = BeautifulSoup(page.text, "html.parser")

#####################
full_name = soup.find("span", class_="user-full-name").text
first_name = soup.find("span", class_="user-name-first").text
last_name = soup.find("span", class_="user-name-last").text

specialty = soup.find("a", class_="profile-head-subtitle").text
city_state = soup.find("span", class_="profile-head-subtitle").text
subspecialty = soup.find("p", class_="user-subspecialty").text
job_title = soup.find("p", class_="user-job-title").text

st_address = soup.find("span", class_="black profile-contact-labels-wrap").text
PRE_phone_num = soup.find("div", class_="profile-contact-labels-wrap")
phone_num = PRE_phone_num.find("span", class_="black").text

ed_trn_elements = soup.find("ul", class_="profile-sectioned-list training").find_all("div", class_="profile-section-wrapper-text")
for element in ed_trn_elements:
    institution = element.find("span", class_="black").text
    education = element.find("span", class_="br").text
    print(institution)
    print(education)
    print()

########################
# print(full_name)
# print(first_name)
# print(last_name)
# print(specialty)
# print(city_state)
# print(subspecialty)
# print(job_title)
# print(st_address)
# print(phone_num)
radiologists = ["<NAME>", "<NAME>", "<NAME>"]
for rad in radiologists:
    print(rad.replace(' ', '-').lower())
# Import data
scraper_list = pd.read_csv(r"C:\Users\ssantavicca3\Documents\Work Files & Folders\RadiologyTrainees_NotABot\scraper_list.csv")
scraper_list.head()
for index, rad in scraper_list[0:5].iterrows():
    print(rad["first_nm"])
   
# (scraper_list["first_nm"][0]+"-"+scraper_list["last_nm"][0]).lower()
# scraper_list[0:2].iat[0,0]
for index, rad in scraper_list[0:5].iterrows():
    npi = rad["NPI"]
    input_name = (rad["first_nm"]+" "+rad["last_nm"]).replace(' ', '-').lower()
#     input_name = rad.replace(' ', '-').lower()
    print(npi)
    print(input_name)
    
# Initialize request details and lists to dump data dicts
prefix = 'https://www.doximity.com/pub/'
user_agent = {'User-agent' : 'Mozilla/5.0'}
verif_data = []
ed_trn_data = []

# Loop through each radiologist and scrape data of interest
with elapsed_timer() as elapsed:
    n_iter = 0
    
    for index, rad in scraper_list.iterrows(): 
        npi = rad["NPI"]
        input_name = (rad["first_nm"]+" "+rad["last_nm"]).replace(' ', '-').lower()  # replace() covers two part last names (e.g., De Soto)
        
        # Beautiful Soup
        try:
            degree = "md"
            url = prefix+input_name+"-"+degree
            page = requests.get(url, headers=user_agent)
        except:
            # Account for DOs, or skip iteration if no hits at all
            try:
                degree = "do"
                url = prefix+input_name+"-"+degree
                page = requests.get(url, headers=user_agent)
            except:
                continue
                
        soup = BeautifulSoup(page.text, "html.parser")

        ## Verification data
        verif_details = {}
        verif_details['NPI'] = npi  #ID
        verif_details['url'] = url

        # Parse out exact text we want & add data to the dictionary
        try:
            verif_details['full_name'] = soup.find("span", class_="user-full-name").text
        except:
            pass

        try:
            verif_details['first_name'] = soup.find("span", class_="user-name-first").text
            verif_details['last_name'] = soup.find("span", class_="user-name-last").text
            verif_details['credentials'] = soup.find("span", class_="user-name-credentials").text
        except:
            pass
        
        try:
            verif_details['specialty'] = soup.find("a", class_="profile-head-subtitle").text
        except:
            pass

        try:
            city_state = soup.find("span", class_="profile-head-subtitle").text
            verif_details['city_state'] = city_state[2:]   #remove "/.n" from beginning of these strings
        except:
            pass

        try:
            verif_details['subspecialty'] = soup.find("p", class_="user-subspecialty").text
        except:
            pass

        try:
            verif_details['job_title'] = soup.find("p", class_="user-job-title").text
        except:
            pass

        try:
            verif_details['st_address'] = soup.find(itemprop="streetAddress").get("content")
            verif_details['zip'] = soup.find(itemprop="postalCode").get("content") 
            verif_details['st_address_full'] = soup.find("span", class_="black profile-contact-labels-wrap").text
        except:
            pass

        try:
            verif_details['phone_num'] = soup.find("div", class_="profile-contact-labels-wrap").find("span", class_="black").text
        except:
            pass
        
        # Append the dictionaries to the list
        verif_details['idx_scraper_list'] = index  
        verif_data.append(verif_details)

        ## Education and training data
        try:
            ed_trn_elements = soup.find("ul", class_="profile-sectioned-list training").find_all("div", class_="profile-section-wrapper-text")
            for element in ed_trn_elements:
                ed_trn_details = {}
                ed_trn_details['NPI'] = npi  #ID

                # Parse out exact text we want & add data to the dictionary
                try:
                    ed_trn_details['institution'] = element.find("span", class_="black").text
                except:
                    pass

                try:
                    ed_trn_details['education'] = element.find("span", class_="br").text
                except:
                    pass

                # Append the dictionaries to the list
                ed_trn_details['idx_scraper_list'] = index
                ed_trn_data.append(ed_trn_details)
        except:
            pass
        
        ## Counter and timer for progress checks  
        n_iter += 1
        if n_iter % 100 == 0:
            print("Iteration (radiologists): "+str(n_iter)+" ----- Time Elapsed: "+str(timedelta(seconds=round(elapsed()))))
            sys.stdout.flush()

            # Save intermediate output incase of crash or timeout
            filename1 = "datadump/doximity dump/saved_ed_trn_list.txt"
            with open(filename1, 'w') as f:
                for item in ed_trn_data:
                    f.write(f'{item}\n')
                    sys.stdout.flush()
            filename2 = "datadump/doximity dump/saved_verif_list.txt"
            with open(filename2, 'w') as f:
                for item in verif_data:
                    f.write(f'{item}\n')
                    sys.stdout.flush()

        # Iteration delay
        sleep(randint(1,2))
## Create and format a DF from the list of dictionaries
ed_trn_df = pd.DataFrame.from_dict(ed_trn_data)
verif_df = pd.DataFrame.from_dict(verif_data)

len(ed_trn_df)
len(verif_df)<jupyter_output><empty_output><jupyter_text>Just need to run loop on full list of radiologists.  Before doing so, maybe run on first 25-50 rads to (1) get an estimate of ETA, and (2) determine whether there are any bugs that need to be accounted for before full run (e.g., maybe a try/except for MDs/DOs/no page at all). May also need to do a follow up scraper with google searches (e.g., "<NAME> MD doximity radiology") do get the profiles of people with either NaN or non-radiology specialties that likely shared name with multiple other physicians. Eg:https://www.doximity.com/pub/michael-fishman-md-ff5f620ahttps://www.doximity.com/pub/michael-fishman-md-acf558dehttps://www.doximity.com/pub/michael-fishman-md-8b21aefd EXPLORE THE MISSING DATA MORE FIRST  i.e., it looks like there are plenty of urls that work but didn't pick up any information (at least in the verif_df file); it even looks like they're missing at consistent intervals??? For the 11,000 or so from the ed_trn_df file, this doesn't appear to be an issue?<jupyter_code># Code to read back in the .txt files created above (i.e., if I close the kernal before saving the local files created above)
from ast import literal_eval

def txt_to_dict(filename):
    tmp_data = list()
    with open(filename, 'r', encoding='cp1252') as f:
        rows = f.readlines()
        for row in rows:
            row = literal_eval(row)
            tmp_data.append(row)
    return tmp_data
ed_trn_dict = txt_to_dict(r"C:\Users\ssantavicca3\Documents\Work Files & Folders\RadiologyTrainees_NotABot\datadump\doximity dump\saved_ed_trn_list.txt")
verif_dict = txt_to_dict(r"C:\Users\ssantavicca3\Documents\Work Files & Folders\RadiologyTrainees_NotABot\datadump\doximity dump\saved_verif_list.txt")

ed_trn_df = pd.DataFrame.from_dict(ed_trn_dict) 
verif_df = pd.DataFrame.from_dict(verif_dict)

ed_trn_df.to_csv(r"C:\Users\ssantavicca3\Documents\Work Files & Folders\RadiologyTrainees_NotABot\ed_trn_df.csv", index=False)
verif_df.to_csv(r"C:\Users\ssantavicca3\Documents\Work Files & Folders\RadiologyTrainees_NotABot\verif_df.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1345764517847629, 'fraction_numerical': 0.006286627597229622, 'mean_word_length': 2.8657331136738056, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 5, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '59043', 'license': 'MIT', 'n_tokens_mistral': 3078, 'n_tokens_neox': 2972, 'n_words': 645, 'path': 'DoximityScraper.ipynb', 'repo_name': 'ssantavicca3/RadiologyTrainees_NotABot', 'size': '9446'}