<jupyter_start><jupyter_text>Given that the RIAA data is hard to obtain, we have to create a Selenium script and we create the following function to fetch the data<jupyter_code>from selenium import webdriver
from time import sleep

driver = webdriver.Chrome('/Users/iliasmiraoui/Desktop/chromedriver')
def get_riaa(artist):
    query = artist
    query = urllib.parse.quote_plus(query)
    df = pd.DataFrame(columns=["artist","artists","songs","last_certification","label","format","details"])
    url = "https://www.riaa.com/gold-platinum/?tab_active=default-award&ar="+query+"&lab=&genre=&format=&date_option=release&from=&to=&award=&type=&category=&adv=SEARCH#search_section"
    driver.get(url)
    sleep(0.1)
    artists = []
    songs = []
    last_certification = []
    label = []
    frmat= []
    details = []
    hascontent =0
    try:
        driver.find_element_by_xpath("//*[@id='loadmore']").click()
    except:
        hascontent =1
        sleep(0.3)
    to_click = driver.find_elements_by_xpath("//*[@class='others_cell format_cell']/div/a")
    for link in to_click:
        link.click()
        sleep(0.6)
    awards = driver.find_elements_by_xpath("//*[@class='table_award_row expanded']")
    if hascontent ==1:
        awards.extend(driver.find_elements_by_xpath("//*[@class='table_award_row expanded hasContent']"))
    nb_songs = len(awards)
    for award in awards:
        artists.append(award.find_element_by_xpath(".//*[@class='artists_cell']").text)
        songs.append(award.find_element_by_xpath(".//*[@class='others_cell']").text)
        last_certification.append(award.find_elements_by_xpath(".//*[@class='others_cell']")[1].text)
        label.append(award.find_elements_by_xpath(".//*[@class='others_cell']")[2].text)
        frmat.append(award.find_element_by_xpath(".//*[@class='others_cell format_cell']").text)
        next_element = award.find_element_by_xpath(".//following-sibling::tr")
        detail = next_element.find_element_by_xpath("//*[@class='table_award_row expanded']")
        details.append([x.text for x in next_element.find_elements_by_xpath(".//*[@class='content_recent_table']/td")])
    for i in range(nb_songs):
        df.loc[i] = [artist,artists[i],songs[i],last_certification[i],label[i],frmat[i],details[i]]
    return df<jupyter_output><empty_output><jupyter_text>We import our datasets and start getting the data. (This took ~15hours to run)<jupyter_code>import pandas as pd
import urllib.parse
import pickle 


with open('song_df_100.pkl', 'rb') as f:
        song_df = pickle.load(f)

from tqdm import tqdm_notebook as tqdm
driver = webdriver.Chrome('/Users/iliasmiraoui/Desktop/chromedriver')
pbar = tqdm(song_df["artist"].unique())
i = 0
for artist in pbar:
    if (artist not in artist_df["artist"].unique()) and (artist not in artist_df[0].unique()):
        i=i+1
        df = get_riaa(artist)
        artist_df = artist_df.append(df)
        if len(df) == 0:
            artist_df = artist_df.append([artist,None,None,None,None,None,None])
        print(i)
        if i % 5 == 0:
            with open('artist_df.pkl', 'wb') as f:
                pickle.dump(artist_df, f)<jupyter_output><empty_output><jupyter_text>Similarly to the Spotify Data, the RIAA search function is very peculiar and will only accept very similar strings. We separate the artists accordingly and we fetch those too.<jupyter_code>import numpy as np
keywords = ["Featuring","Feauring", "featuring", "&",","," X ", " x ","Presents", "From", "Or","And",]
new_df = artist_df.dropna(subset=[0])
temp_list = np.asarray(new_df[0])
count=0
for keyword in keywords:
    artist_list = [i.split(keyword) for i in temp_list]
    temp_list = list()
    count=count+1
    for i in artist_list:
            temp_list.extend(i)

artist_list = list()
for i in temp_list:
    i = i.strip()
    if len(i)>1:
        artist_list.append(i)
artist_list = set(artist_list)        
artist_list_new = set(artist_list) 

for i in artist_list:
    if len(artist_df[artist_df[0] == i]) >0 and i in artist_list_new:
        artist_list_new.remove(i)
    elif len(artist_df[artist_df["artist"] == i]) >0 and i in artist_list_new:
        artist_list_new.remove(i)
    
from tqdm import tqdm_notebook as tqdm
driver = webdriver.Chrome('/Users/iliasmiraoui/Desktop/chromedriver')
pbar = list(artist_list_new)
i = 0
for artist in pbar:
    if (artist not in artist_df["artist"].unique()) and (artist not in artist_df[0].unique()):
        i=i+1
        df = get_riaa(artist)
        artist_df = artist_df.append(df)
        if len(df) == 0:
            artist_df = artist_df.append([artist,None,None,None,None,None,None])
        print(i)
        if i % 5 == 0:
            with open('artist_df.pkl', 'wb') as f:
                pickle.dump(artist_df, f)
with open('artist_df.pkl', 'wb') as f:
    pickle.dump(artist_df, f)<jupyter_output><empty_output><jupyter_text>Let's clean the dataset where the rows are empty<jupyter_code>artist_df = artist_df.drop([0],axis=1)  #delete first column, no information
artist_df = artist_df.replace(to_replace='None', value=np.nan).dropna() 
artist_df = artist_df.reset_index()
artist_df.pop("index")
artist_df<jupyter_output><empty_output><jupyter_text>Given that the search function is very peculiar, we check our dataset and clean it accordingly. The data is messy, VERY MESSY. So we look at the messy data points by varying the thresholds below and we manage to clean it by removing the rows as per the functions below.<jupyter_code>from fuzzywuzzy import fuzz

def similar(a, b, threshold=70):
    return (fuzz.partial_ratio(a, b) >= threshold)

for row in artist_df.index:
    if not similar(artist_df.loc[row,"artist"].upper(),artist_df.loc[row,"artists"]) and (similar(artist_df.loc[row,"artist"].upper(),artist_df.loc[row,"artists"],95)) and (fuzz.ratio(artist_df.loc[row,"artist"].upper(),artist_df.loc[row,"artists"]) <=95):
        print(fuzz.partial_ratio(artist_df.loc[row,"artist"].upper(),artist_df.loc[row,"artists"]))
        print(row, artist_df.loc[row,"artist"],artist_df.loc[row,"artists"])   

to_drop = [346,354,360,1831,1832,1833,1835,1842,1844,1846,1849,1850,1851,1855,1856,2441,2443,2447,229,2450,2452,
          2849,3278,3282,3286,3296,3297,3298,3299,3300,3301,3302,3523,3590,3604,3765,3769,3809,3810,3811,4057,4079,
           4083,4086,4194,4198,4199,4209,4213,4216,4303,4404,4458,4534,5260,5262,5572,5793,5794,5795,5916,5918,5919,
           5920,5922,6049,6050,6067,6240,6253,6254,6255,6276,6279,6278,6280,6283,6287,6286,6289,6290,6291,6293,6306,
           6815,7053,7056,7271,7272,7273,7274,7597,7598,7599,7600,7791,7793,7794,7795,7798,7799,7800,7804,7805,7812,7818,
           8033,8035,8036,8037,8092,8142,8145,8147,8259,8284,8610,8611,8666,8670,8671,8672,8675,8676,8678,8683,8753,8785,8786,8787,
           8799,8800,8809,8903,8908,8909,8912,8926,8927,9231,9233,9289,9290,9310,9315,9394,9396,9512,9567,9568,9569,9571,
           9602,9627,9628,9771,9796,9799,9800,9846,9848,9851,9891,9892,9893,9894,9957,9959,10043,10044,10169,10170,10204,
           10205,10206,10328,10329,10505,10506,10529,10567,10573,10624,10695,10745,10746,10915,10989,10995,11100,11133,
           11141,11144,11234,11297,11298,11341,11348,11520,11521,11522,11624,11625,11626,11807,11808,11809,11814,11817,
           12298,12495,12501,12502,12503,12931,12936,12937,
          ]

to_drop.extend([i for i in range(13342,13372)])
to_drop.extend([i for i in range(13268,13309)])
to_drop.extend([i for i in range(13233,13259)])
to_drop.extend([i for i in range(13191,13232)])
to_drop.extend([i for i in range(13159,13187)])
to_drop.extend([i for i in range(13147,13158)])
to_drop.extend([i for i in range(13105,13135)])
to_drop.extend([i for i in range(13061,13086)])
to_drop.extend([i for i in range(13013,13043)])
to_drop.extend([i for i in range(12951,12999)])
to_drop.extend([i for i in range(12907,12930)])
to_drop.extend([i for i in range(12895,12903)])
to_drop.extend([i for i in range(12881,12894)])
to_drop.extend([i for i in range(12844,12855)])
to_drop.extend([i for i in range(12813,12843)])
to_drop.extend([i for i in range(12758,12794)])
to_drop.extend([i for i in range(12743,12756)])
to_drop.extend([i for i in range(12694,12740)])
to_drop.extend([i for i in range(12643,12672)])
to_drop.extend([i for i in range(12592,12626)])
to_drop.extend([i for i in range(12547,12577)])
to_drop.extend([i for i in range(12458,12486)])
to_drop.extend([i for i in range(12391,12420)])
to_drop.extend([i for i in range(12374,12377)])
to_drop.extend([i for i in range(12303,12364)])
to_drop.extend([i for i in range(12239,12269)])
to_drop.extend([i for i in range(12196,12226)])
to_drop.extend([i for i in range(12146,12176)])
to_drop.extend([i for i in range(12081,12111)])
to_drop.extend([i for i in range(12027,12079)])
to_drop.extend([i for i in range(12017,12025)])
to_drop.extend([i for i in range(11986,11997)])
to_drop.extend([i for i in range(11945,11980)])
to_drop.extend([i for i in range(11904,11935)])
to_drop.extend([i for i in range(11868,11887)])
to_drop.extend([i for i in range(11831,11860)])
to_drop.extend([i for i in range(11818,11827)])
to_drop.extend([i for i in range(11810,11813)])
to_drop.extend([i for i in range(11776,11798)])
to_drop.extend([i for i in range(11773,11775)])
to_drop.extend([i for i in range(11593,11623)])
to_drop.extend([i for i in range(11547,11549)])
to_drop.extend([i for i in range(11537,11545)])
to_drop.extend([i for i in range(11527,11534)])
to_drop.extend([i for i in range(11484,11503)])
to_drop.extend([i for i in range(11407,11467)])
to_drop.extend([i for i in range(11357,11394)])
to_drop.extend([i for i in range(11235,11251)])
to_drop.extend([i for i in range(11147,11184)])
to_drop.extend([i for i in range(11137,11140)])
to_drop.extend([i for i in range(11102,11129)])
to_drop.extend([i for i in range(11086,11098)])
to_drop.extend([i for i in range(11065,11083)])
to_drop.extend([i for i in range(11000,11011)])
to_drop.extend([i for i in range(10997,10999)])
to_drop.extend([i for i in range(10917,10931)])
to_drop.extend([i for i in range(10877,10882)])
to_drop.extend([i for i in range(10822,10829)])
to_drop.extend([i for i in range(10809,10815)])
to_drop.extend([i for i in range(10767,10796)])
to_drop.extend([i for i in range(10749,10758)])
to_drop.extend([i for i in range(10709,10743)])
to_drop.extend([i for i in range(10699,10707)])
to_drop.extend([i for i in range(10658,10686)])
to_drop.extend([i for i in range(10625,10639)])
to_drop.extend([i for i in range(10593,10622)])
to_drop.extend([i for i in range(10475,10496)])
to_drop.extend([i for i in range(10445,10451)])
to_drop.extend([i for i in range(10331,10340)])
to_drop.extend([i for i in range(10296,10324)])
to_drop.extend([i for i in range(10269,10277)])
to_drop.extend([i for i in range(10237,10267)])
to_drop.extend([i for i in range(10109,10139)])
to_drop.extend([i for i in range(10079,10108)])
to_drop.extend([i for i in range(10045,10072)])
to_drop.extend([i for i in range(10010,10040)])
to_drop.extend([i for i in range(9984,9995)])
to_drop.extend([i for i in range(9940,9944)])
to_drop.extend([i for i in range(9913,9935)])
to_drop.extend([i for i in range(9852,9882)])
to_drop.extend([i for i in range(9712,9740)])
to_drop.extend([i for i in range(9539,9562)])
to_drop.extend([i for i in range(9438,9468)])
to_drop.extend([i for i in range(9316,9346)])
to_drop.extend([i for i in range(9291,9300)])
to_drop.extend([i for i in range(9250,9280)])
to_drop.extend([i for i in range(9197,9201)])
to_drop.extend([i for i in range(9076,9080)])
to_drop.extend([i for i in range(9006,9011)])
to_drop.extend([i for i in range(8877,8892)])
to_drop.extend([i for i in range(362,376)])
to_drop.extend([i for i in range(1836,1830)])
to_drop.extend([i for i in range(1857,1861)])
to_drop.extend([i for i in range(2427,2430)])
to_drop.extend([i for i in range(2427,2430)])
to_drop.extend([i for i in range(2803,2808)])
to_drop.extend([i for i in range(2427,2430)])
to_drop.extend([i for i in range(2962,2967)])
to_drop.extend([i for i in range(3481,3493)])
to_drop.extend([i for i in range(3591,3602)])
to_drop.extend([i for i in range(3607,3620)])
to_drop.extend([i for i in range(3812,3838)])
to_drop.extend([i for i in range(4058,4073)])
to_drop.extend([i for i in range(4268,4276)])
to_drop.extend([i for i in range(4277,4297)])
to_drop.extend([i for i in range(4303,4332)])
to_drop.extend([i for i in range(4410,4425)])
to_drop.extend([i for i in range(5064,5069)])
to_drop.extend([i for i in range(5574,5589)])
to_drop.extend([i for i in range(5873,5903)])
to_drop.extend([i for i in range(5923,5931)])
to_drop.extend([i for i in range(6296,6305)])
to_drop.extend([i for i in range(6310,6322)])
to_drop.extend([i for i in range(6787,6791)])
to_drop.extend([i for i in range(6822,6827)])
to_drop.extend([i for i in range(6832,6862)])
to_drop.extend([i for i in range(7009,7039)])
to_drop.extend([i for i in range(7057,7065)])
to_drop.extend([i for i in range(7066,7096)])
to_drop.extend([i for i in range(7136,7165)])
to_drop.extend([i for i in range(7257,7270)])
to_drop.extend([i for i in range(7779,7790)])
to_drop.extend([i for i in range(7808,7812)])
to_drop.extend([i for i in range(7822,7831)])
to_drop.extend([i for i in range(7840,7843)])
to_drop.extend([i for i in range(7846,7863)])
to_drop.extend([i for i in range(7871,7873)])
to_drop.extend([i for i in range(7876,7899)])
to_drop.extend([i for i in range(7257,7270)])
to_drop.extend([i for i in range(8148,8153)])
to_drop.extend([i for i in range(8264,8274)])
to_drop.extend([i for i in range(8396,8411)])
to_drop.extend([i for i in range(8418,8437)])
to_drop.extend([i for i in range(8438,8441)])
to_drop.extend([i for i in range(8442,8447)])
to_drop.extend([i for i in range(8495,8503)])
to_drop.extend([i for i in range(8506,8523)])
to_drop.extend([i for i in range(8653,8665)])
to_drop.extend([i for i in range(8792,8797)])
to_drop.extend([i for i in range(8815,8840)])

for i in to_drop:
    if i in artist_df.index:
        artist_df =artist_df.drop(i)

to_drop = [244,3772,4554,4726,6124,6126,6127,7391,7591,8000,8001,8002,8649,8910,8911,8928,10876,11101,11349,11350,11353,
          11507,11508,11509,11656,12283,12425]

to_drop.extend([i for i in range(4649,4653)])
to_drop.extend([i for i in range(4730,4736)])
to_drop.extend([i for i in range(5843,5873)])
to_drop.extend([i for i in range(7990,7999)])
to_drop.extend([i for i in range(8904,8908)])
to_drop.extend([i for i in range(8913,8917)])
to_drop.extend([i for i in range(9067,9069)])
to_drop.extend([i for i in range(10370,10386)])
to_drop.extend([i for i in range(11026,11056)])
to_drop.extend([i for i in range(11658,11686)])
to_drop.extend([i for i in range(11827,11830)])
to_drop.extend([i for i in range(11892,11901)])

for i in to_drop:
    if i in artist_df.index:
        artist_df =artist_df.drop(i)

to_drop = [3931,303,4425,4426,5532,4433,4434,7588,7761,8900,8901,8902,8949,9061,9346,9347,9348,9350,9351,9356,9357,9958,
          9960,9961,10882,10883,10994,10996,10999,11011,11352,11354,11714,11716,11721,13066,13067]

to_drop.extend([i for i in range(3908,3917)])
to_drop.extend([i for i in range(3923,3930)])
to_drop.extend([i for i in range(3933,3937)])
to_drop.extend([i for i in range(7593,7597)])
to_drop.extend([i for i in range(8644,8653)])
to_drop.extend([i for i in range(8917,8922)])
to_drop.extend([i for i in range(9362,9376)])
to_drop.extend([i for i in range(10229,10235)])
to_drop.extend([i for i in range(11012,11019)])
to_drop.extend([i for i in range(11342,11348)])
to_drop.extend([i for i in range(11693,11697)])
to_drop.extend([i for i in range(11269,11297)])

for i in to_drop:
    if i in artist_df.index:
        artist_df =artist_df.drop(i)<jupyter_output><empty_output><jupyter_text>We now clean the RIAA output into clear "Platinum" "Gold" and number of award for each date an award is given for a song and we save the dataframe for further processing<jupyter_code>n_df = pd.DataFrame(columns=["artist","artists","award","award_date","format","label","last_certification","songs"])
df2 = artist_df
l = df2['details'].tolist()
for i in range(len(df2)):
    certif = ['Platinum', 'Gold']
    new_l = []
    for j in range(len(l[i])):
        for word in certif:
            if word in l[i][j]:
                new_l.append(l[i][j])
                artist = df2.iloc[i][0]
                artists = df2.iloc[i][1]
                award = l[i][j].split("|")[0]
                award_date = l[i][j].split("|")[1]
                label = df2.iloc[i][4]
                formats = df2.iloc[i][3]
                last_certif = df2.iloc[i][5]
                songs = df2.iloc[i][6]
                n_df.loc[len(n_df)] = [artist,artists,award,award_date,formats,label,last_certif,songs]
for row in n_df.index:
    award = n_df.loc[row,"award"].split("x ")
    if len(award)>1:
        n_df.loc[row,"award"] = "Platinum"
        n_df.loc[row,"award_num"] = int(award[0])
    else:
        n_df.loc[row,"award_num"] = 1
with open('riaa_df.pkl', 'wb') as f:
    pickle.dump(n_df, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.159772674009843, 'fraction_numerical': 0.1603585657370518, 'mean_word_length': 5.1732368896925855, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '359227', 'license': 'MIT', 'n_tokens_mistral': 8443, 'n_tokens_neox': 6926, 'n_words': 1399, 'path': 'Scraping/ScrapeRIAA.ipynb', 'repo_name': 'imiraoui/billboard-analytics', 'size': '17068'}