<jupyter_start><jupyter_text>First fetch Billboard top 100Uses RSS feed reader in this format:```  1: Bad And Boujee  http://www.billboard.com/charts/hot-100  Migos Featuring Lil Uzi Vert  Bad And Boujee  1  1  Bad And Boujee by Migos Featuring Lil Uzi Vert ranks 1  The Hot 100  Sat, 11 Feb 2017 00:00:00 -0500  http://www.billboard.com/charts/hot-100 ```<jupyter_code># Fetch the Billboard Hot 100 for this week

import requests
from lxml import etree

def get_billboard_hot_100():
    hot_100_rss = requests_get('http://www.billboard.com/rss/charts/hot-100')
    xml = bytes(bytearray(hot_100_rss.text, encoding='utf-8'))
    tree = etree.XML(xml)

    songs = []
    items = tree.xpath("//item")
    for item in items:
        song = {}

        song['title'] = item.xpath("chart_item_title/text()")[0]
        song['artist'] = item.xpath("artist/text()")[0]
        song['rank_this_week'] = item.xpath("rank_this_week/text()")[0]
        song['rank_last_week'] = item.xpath("rank_last_week/text()")[0]

        songs.append(song)
        
    return songs

songs = get_billboard_hot_100()
songs<jupyter_output>Error occurred during formatting data. Trying to use fallback server https://fake-useragent.herokuapp.com/browsers/0.1.4
Traceback (most recent call last):
  File "/software/miniconda3/lib/python3.5/urllib/request.py", line 1254, in do_open
    h.request(req.get_method(), req.selector, req.data, headers)
  File "/software/miniconda3/lib/python3.5/http/client.py", line 1106, in request
    self._send_request(method, url, body, headers)
  File "/software/miniconda3/lib/python3.5/http/client.py", line 1151, in _send_request
    self.endheaders(body)
  File "/software/miniconda3/lib/python3.5/http/client.py", line 1102, in endheaders
    self._send_output(message_body)
  File "/software/miniconda3/lib/python3.5/http/client.py", line 934, in _send_output
    self.send(msg)
  File "/software/miniconda3/lib/python3.5/http/client.py", line 877, in send
    self.connect()
  File "/software/miniconda3/lib/python3.5/http/client.py", line 849, in connect
    (self.host,self.port), self.timeout, se[...]<jupyter_text>Test scraping lyrics from AZLyrics.comhttp://search.azlyrics.com/search.php?q=Bad+And+Boujee+Migos+Lil+Uzi+VertInitial code borrowed from:https://github.com/blha303/getlyricsThis method works, however you must be cautious in how many requests you perform in a short time frame. Otherwise you might get your IP permanently blocked from the site.<jupyter_code>from urllib.parse import urlencode
from lxml import html
from lxml.etree import tostring

def get_az_lyrics(search):   
    # search the AZLyrics site and scrape results
    base_url = 'http://search.azlyrics.com/search.php?'
    full_url = base_url + urlencode({'q': search})
    
    result = requests_get(full_url)
    tree = html.fromstring(result.text)
    
    # Look for all URLs in search results
    # When no results - return None
    search_results = tree.xpath("//td/a[@target='_blank']")
    if not search_results:
        return None
    
    # Scrape the lyrics page for the lyrics
    lyrics_url = search_results[0].get('href')
    result = requests_get(lyrics_url)
    tree = html.fromstring(result.text)
    
    lyrics_html = tree.xpath("//div")
    return lyrics_html[22].text_content()

lyrics = get_az_lyrics("Bad And Boujee Migos Lil Uzi Vert")
print(lyrics)
import time

missing = 0
total = 0

# Scrape all lyrics for all songs
for song in songs:
    search_criteria = song['title'] + ' ' + song['artist'].replace('Featuring', '')
    song['lyrics'] = get_az_lyrics(search_criteria)
    
    time.sleep(1)
    
    if not song['lyrics']:
        missing = missing + 1
        
    total = total + 1
    
    if total >= 5:
        break
        
print(missing)<jupyter_output><empty_output><jupyter_text>Scrape Lyrics from LyricsFreakhttp://www.lyricsfreak.com/search.php?a=search&type=song&q=Bad+And+BoujeeThis site only really lets you search by song or artist only. That is odd...<jupyter_code>from urllib.parse import urlencode
from lxml import html
from lxml.etree import tostring
import re

def get_lyrics_freak(artist, song):
    def primary_artist(artist):
        artist = artist.casefold()
        artist = artist.split('featuring')[0].strip()
        artist = artist.split('/')[0].strip()
        artist = artist.split(' x ')[0].strip()

        return artist
    
    artist = primary_artist(artist)
    
    # search the LyricsFreak site and scrape results
    base_url = 'http://www.lyricsfreak.com'
    full_url = base_url + '/search.php?' + urlencode({'a': 'search', 'type': 'song', 'q': song})
    
    result = requests_get(full_url)
    tree = html.fromstring(result.text)
    
    # Look for all URLs in search results
    # When no results - return None
    search_results = tree.xpath("//td/a")
    if not search_results:
        return None
    
    # find the lyrics page...
    previous_is_artist = False
    lyrics_url = None
    for result in search_results:
        current_text = result.text_content().casefold()
        if previous_is_artist and song.casefold() in current_text:
            lyrics_url = base_url + result.get("href")
            break        
        
        previous_is_artist = artist.casefold() in current_text
    
    if not lyrics_url:
        return None
    
    # scrape the lyrics
    result = requests_get(lyrics_url)
    tree = html.fromstring(result.text)
    
    lyric_html = tree.xpath("//div[@id='content_h']")
    if not lyric_html:
        return None
    
    lyric_html = str(tostring(lyric_html[0]))
    return html.fromstring(re.sub(r'<br\w*\/?>', r'\\n', lyric_html)).text_content()

print(get_lyrics_freak("Migos", "Bad And Boujee"))
import time

missing = 0
total = 0

# Scrape all lyrics for all songs
    song['lyrics'] = get_lyrics_freak(song['artist'], song['title'])
    
    time.sleep(1)
    
    if not song['lyrics']:
        missing = missing + 1
        
print(missing)
for song in songs:
    if song['lyrics'] is None:
        print(song['artist'] + ' '+ song['title'])<jupyter_output><NAME> Shape Of You
The Chainsmokers Featuring Halsey Closer
The Weeknd Featuring Daft Punk Starboy
The Chainsmokers Paris
Big Sean Bounce Back
DJ Snake Featuring Justin Bieber Let Me Love You
<NAME>
The Weeknd Featuring Daft Punk I Feel It Coming
Sh<NAME>
D.R.A.M. Featuring Lil Yachty Broccoli
KYLE Featuring Lil Yachty iSpy
<NAME> & <NAME>Call Juju On That Beat (TZ Anthem)
Clean Bandit Featuring Sean Paul & Anne-Marie Rockabye
The Chainsmokers Featuring Daya Don't Let Me Down
Little Big Town Better Man
Migos T-Shirt
<NAME> Featuring Usher & Gucci Mane Party
21 Savage & Metro Boomin Featuring Future X
Bl<NAME> A Guy With A Girl
<NAME> Dirt On My Boots
<NAME>
Future Featuring Drake Used To This
Gucci Mane Featuring Drake Both
The Weeknd Party Monster
<NAME>ill Down
Kaleo Way Down We Go
<NAME> Seein' Red
Bebe Rexha I Got You
Noah Cyrus Featuring Labrinth Make Me (Cry)
Post Malone Featuring Quavo Congratulations
21 Savage & Me[...]<jupyter_text>Scrape Lyrics from SongLyricshttp://www.songlyrics.com/index.php?section=search&searchW=The+Chainsmokers+Featuring+Halsey+Closer&submit=Search&searchIn1=artist&searchIn3=song<jupyter_code>from urllib.parse import urlencode
from lxml import html
from lxml.etree import tostring

def get_song_lyrics(artist, song):   
    # search the SongLyrics site and scrape results
    base_url = 'http://www.songlyrics.com'
    params = {
        'section': 'search',
        'searchW': artist + ' ' + song,
        'submit': 'Search',
        'searchIn1': 'artist',
        'searchIn3': 'song'
    }
    full_url = base_url + '/index.php?' + urlencode(params)
    
    result = requests_get(full_url)
    tree = html.fromstring(result.text)
    
    # Look for all URLs in search results
    # When no results - return None
    search_results = tree.xpath("//div[@class='serpresult']/h3/a")
    if not search_results:
        return None
    
    lyrics_url = search_results[0].get('href')
    
    if not lyrics_url:
        return None
    
    # scrape the lyrics
    result = requests_get(lyrics_url)
    tree = html.fromstring(result.text)
    
    lyric_html = tree.xpath("//p[@id='songLyricsDiv']")
    if not lyric_html:
        return None
    
    return lyric_html[0].text_content() 

print(get_song_lyrics("Migos", "Bad And Boujee"))
import time

missing = 0
total = 0

# Scrape all lyrics for all songs
for song in songs:
    song['lyrics'] = get_song_lyrics(song['artist'], song['title'])
    
    time.sleep(1)
    
    if not song['lyrics']:
        missing = missing + 1
        
print(missing)
for song in songs:
    if song['lyrics'] is None:
        song['lyrics'] = get_song_lyrics(song['artist'], song['title'])
        print(song)<jupyter_output>http://www.songlyrics.com/index.php?section=search&searchIn1=artist&searchIn3=song&searchW=Starley+Call+On+Me&submit=Search
http://www.songlyrics.com/starley/call-on-me-lyrics/
{'lyrics': "When you're low\nAnd your knees can't rise\nYou feel helpless\nAnd you're looking to the sky\nSome people would say\nTo accept their fate\nWell, if this is fate\nThen we'll find a way to cheat\n\nCause, oh, we'll say a little prayer\nBut, oh, if the answer isn't there\n\nYou know you can call on me\nWhen you need somebody\nYou know you can call on me\nWhen you can't stop the tears from falling down\nYou know you can call on me\nCall on me, darling\nYou know you can call on me\nCall on me, darling\n\nWhen you're weary\nAnd the road is dark\nAnd I'll guide you\nWith the beating of my heart\nAnd if the calvary\nAnd the help don't come\nWell, then we'll find a way\nTo dodge a smoking gun\n\nCause, oh, we'll say a little prayer\nBut, oh, if the answer isn't there\n\nYou know you can call on me\nWhen you n[...]<jupyter_text>Putting it all together1. Scrape hot 100 billboard songs2. Obtain lyrics for each song3. Save song list to csv file4. Save lyrics to text files<jupyter_code>from slugify import slugify
import os
import time
import csv

def create_dirs():
    dir_path = os.path.dirname(os.path.realpath('.'))
    data_dir = os.path.join(dir_path, 'billboard-hot-100-data')
    save_dir = os.path.join(data_dir, time.strftime("%d-%m-%Y"))
    
    os.makedirs(save_dir, exist_ok=True)
    
    return save_dir


def save_lyrics(save_dir, artist, title, lyrics):
    slugged = slugify(artist + ' ' + title, only_ascii=True) + '.txt'
    save_path = os.path.join(save_dir, slugged)
    
    try:
        f = open(save_path, 'w')
        f.write(lyrics)
    except:
        save_path = ''
    
    return save_path
    

def save_hot_100_and_lyrics():
    
    # create directories to save data to
    save_dir = create_dirs()
    
    # set up csv writer to write songs to
    hot_100_csv_path = os.path.join(save_dir, 'songs.csv')
    hot_100_csv = open(hot_100_csv_path, 'w')
    csv_writer = csv.writer(hot_100_csv, dialect='excel')
    csv_writer.writerow(['artist', 'title', 'rank_this_week', 'rank_last_week', 'lyrics_path'])
    
    for song in get_billboard_hot_100():
        
        # first attempt to retrieve song from SongLyrics
        lyrics = get_song_lyrics(song['artist'], song['title'])
        
        # try LyricsFreak
        if lyrics is None:
            lyrics = get_lyrics_freak(primary_artist(song['artist']), song['title'])
        
        if lyrics is None:
            raise "Unable to fetch lyrics for: " + song['artist'] + " " + song['title']
            
        # save lyrics to file and write song to csv
        lyric_file_path = save_lyrics(save_dir, song['artist'], song['title'], lyrics)
        csv_writer.writerow([
            song['artist'],
            song['title'],
            song['rank_this_week'],
            song['rank_last_week'],
            lyric_file_path
        ])
        
    hot_100_csv.close()
    
save_hot_100_and_lyrics()<jupyter_output>http://www.songlyrics.com/index.php?searchW=Migos+Featuring+Lil+Uzi+Vert+Bad+And+Boujee&searchIn1=artist&submit=Search&section=search&searchIn3=song
http://www.songlyrics.com/migos/bad-and-boujee-feat-lil-uzi-vert-lyrics/
http://www.songlyrics.com/index.php?searchW=Ed+Sheeran+Shape+Of+You&searchIn1=artist&submit=Search&section=search&searchIn3=song
http://www.songlyrics.com/ed-sheeran/shape-of-you-lyrics/
http://www.songlyrics.com/index.php?searchW=The+Chainsmokers+Featuring+Halsey+Closer&searchIn1=artist&submit=Search&section=search&searchIn3=song
http://www.songlyrics.com/the-chainsmokers-feat-halsey/closer-feat-halsey-lyrics/
http://www.songlyrics.com/index.php?searchW=Machine+Gun+Kelly+x+Camila+Cabello+Bad+Things&searchIn1=artist&submit=Search&section=search&searchIn3=song
http://www.songlyrics.com/machine-gun-kelly-feat-camila-cabello/bad-things-lyrics/
http://www.songlyrics.com/index.php?searchW=Zayn+%2F+Taylor+Swift+I+Don%27t+Wanna+Live+Forever+%28Fifty+Shades+Darker%29&searchIn[...]
{'fraction_non_alphanumeric': 0.11750580945003873, 'fraction_numerical': 0.01471727343144849, 'mean_word_length': 3.495473537604457, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 2, 'lorem ipsum': 0, 'www.': 18, 'xml': 9}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '563737', 'license': 'MIT', 'n_tokens_mistral': 4658, 'n_tokens_neox': 4392, 'n_words': 1075, 'path': 'notebooks/exploratory/01-web-scraping.ipynb', 'repo_name': 'tylerwmarrs/billboard-hot-100-lyric-analysis', 'size': '12973'}