<jupyter_start><jupyter_text>Lyrics Project<jupyter_code>import requests
from bs4 import BeautifulSoup
import re<jupyter_output><empty_output><jupyter_text>Access the Artist URL<jupyter_code>url = 'https://www.lyrics.com/artist/R%C3%B3is%C3%ADn-Murphy/350933'
response = requests.get(url)
# Check the status code
response.status_code

# 200: means that the request was valid and we got what we asked for
# 404: Page not found
# Everything with 400 means that we made a mistake or are not authorized
# Everything with 500 means that there is an error on the server side
html_artist = response.text
html_artist[:200]
# retrieves artist name. Includes accentuation and spaces
# re.findall returns a list of strings, hence the indexation.

artist_name = re.findall('Albums by <strong>([^<]+)', html_artist)[0]
artist_name.replace(' ', '_')
parsed_html_artist = BeautifulSoup(html_artist)
type(parsed_html_artist)
html_links = parsed_html_artist.find_all(attrs={'class': 'tal qx'})
len(html_links) # this returns the number of songs
html_links[0]
# This loop uses regular expressions to strip the superfluos information before and after the link.
# re.sub(pattern, replacement, target string): It sub(stitutes) the pattern with an empty substring.
# It also adds the https:... prefix to each link before appending it to a list.

lyrics_links = []
for entry in html_links:
    a = str(entry)
    b = re.sub('.+=+?"', '', a)
    c = re.sub('".+>', '', b)
    c = 'https://www.lyrics.com' + c
    lyrics_links.append(c)
lyrics_links[:3]<jupyter_output><empty_output><jupyter_text>Access the URL of one song<jupyter_code>response_song = requests.get(lyrics_links[0])
response_song.status_code
html_song = response_song.text
soup_song = BeautifulSoup(html_song)
html_lyrics = str(soup_song.find(attrs={'class': "lyric-body"}))
html_lyrics[:200]
# Remove all the noisy links in the lyrics
lyrics = re.sub('<.+?>+?', '', html_lyrics)
# Remove the square brackets before and after the lyrics body
lyrics = re.sub('\[', '', lyrics)
lyrics = re.sub('\]', '', lyrics)
lyrics
# save lyrics as a txt file to the songs folder
with open('./songs/test.txt', 'w') as f:
    f.writelines(lyrics)<jupyter_output><empty_output><jupyter_text>Loop the links and save lyrics as separate files<jupyter_code>'''
Still need to upgrade:
    - a way to extract the name of the artist and encode it in the file and folder. Consider accentuation and spaces!
'''

i = 1
for song in lyrics_links:
    response_song = requests.get(song)
    html_song = response_song.text
    soup_song = BeautifulSoup(html_song)
    html_lyrics = str(soup_song.find(attrs={'class': "lyric-body"}))
    lyrics = re.sub('<.+?>+?', '', html_lyrics)
    lyrics = re.sub('\[', '', lyrics)
    lyrics = re.sub('\]', '', lyrics)
    num_len = len(str(len(lyrics_links))) # lol... read next comment first.
    with open('./songs/Róisín_Murphy_' + str(i).zfill(num_len) + '.txt', 'w') as f:
        #zfill(num_len) makes sure there are enough leading 0's before the number to sort files easily.
        f.writelines(lyrics)
    i += 1<jupyter_output><empty_output><jupyter_text>Define function<jupyter_code>def scrape_lyrics(url):
    '''
    Scrapes the lyrics of all songs of an artist privided their page on lyrics.com.
    The lyrics are then saved as separate files in a new diretory in the current working.
    
    Parameters
    ----------
    url : str
        the url of artist page on lyrics.com
    '''
    response = requests.get(url)
    response.text
    html_artist = response.text
    artist_name = re.findall('Albums by <strong>([^<]+)', html_artist)[0]
    soup_artist = BeautifulSoup(html_artist)
    html_links = soup_artist.find_all(attrs={'class': 'tal qx'})
    lyrics_links = []
    #TODO: song_titles = []
    for entry in html_links:
        a = str(entry)
        b = re.sub('.+=+?"', '', a)
        #TODO: song_title = re.findall('">(.+?)<.+>', a)[0]
        c = re.sub('".+>', '', b)
        d = 'https://www.lyrics.com' + c
        lyrics_links.append(d)
        #TODO: song_titles.append(song_title)
        #TODO: if song_titles: unique...
    i = 1
    for song in lyrics_links:
        response_song = requests.get(song)
        html_song = response_song.text
        # TODO: maybe here I can add the song_name. This may avoid duplicates...
        soup_song = BeautifulSoup(html_song)
        html_lyrics = str(soup_song.find(attrs={'class': "lyric-body"}))
        if html_lyrics == 'None':
            continue # skips empty lyrics
        else:
            lyrics = re.sub('<.+?>+?', '', html_lyrics)
            lyrics = re.sub('\[', '', lyrics)
            lyrics = re.sub('\]', '', lyrics)
            num_len = len(str(len(lyrics_links))) # lol... read next comment first.
            with open('./songs/' + artist_name.replace(' ', '_') + '_' + str(i).zfill(num_len) + '.txt', 'w') as f:
                #zfill(num_len) makes sure there are enough leading 0's before the number to sort files easily.
                f.writelines(lyrics)
                f.read
            i += 1<jupyter_output><empty_output><jupyter_text>Lines for testing code<jupyter_code>scrape_lyrics('https://www.lyrics.com/artist/Diamanda-Gal%C3%A1s/4330')
str(html_links[0])[40:]
re.findall('">(.+?)<.+>', str(html_links[0])[40:])[0]<jupyter_output><empty_output><jupyter_text>Python file<jupyter_code>import requests
from bs4 import BeautifulSoup
import re
import sys
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split as tts
##
from collections import Counter

url = 'https://www.lyrics.com/artist/Arca'
def scrape_links(url):
    '''
    Scrapes the links to all songs of an artist provided their page on lyrics.com.
    
    Parameters
    ----------
    url : str
        the url of artist page on lyrics.com

    Returns
    -------
    list
        a list of strings with the links to the all song lyrics on the artist page on lyrics.com
    '''

    response = requests.get(url)
    html_artist = response.text
    artist_name = re.findall('Albums by <strong>([^<]+)', html_artist)[0]
    soup_artist = BeautifulSoup(html_artist, features="lxml")
    html_links = soup_artist.find_all(attrs={'class': 'tal qx'})
    lyrics_links = []
    song_titles = []
    for entry in html_links:
        str_entry = str(entry.find_all('a')[0])
        path_end, song_title = re.findall('\/l[^<]+', str_entry)[0].split(sep='">')
        if song_title not in song_titles: #this removes duplicates
            lyrics_links.append('https://www.lyrics.com' + path_end)
            song_titles.append(song_title)
    return lyrics_links, artist_name, song_titles
url = 'https://www.lyrics.com/artist/Laurie-Anderson/3545'
len(scrape_links(url)[2])
lyrics_links = scrape_links(url)
lyrics_links
response = requests.get(url)
html_artist = response.text
soup_artist = BeautifulSoup(html_artist, features="lxml")
html_links = soup_artist.find_all(attrs={'class': 'tal qx'})
test = str(html_links[1].find_all('a')[0])
a, b = re.findall('\/l[^<]+', test)[0].split(sep='">')
response = requests.get(url)
html_artist = response.text
re.findall('Albums by <strong>([^<]+)', html_artist)
def scrape_lyrics(url):
    '''
    Scrapes the lyrics of all songs of an artist provided their page on lyrics.com.
    The lyrics are then saved as separate files in a new diretcory in the current working directory.
    
    Parameters
    ----------
    url : str
        the url of artist page on lyrics.com
    '''
    lyrics_links, artist_name = scrape_links(url)
    for i, song in enumerate(lyrics_links, 1):
        response_song = requests.get(song)
        html_song = response_song.text
        # TODO: maybe here I can add the song_name. This may avoid duplicates...
        soup_song = BeautifulSoup(html_song, features="lxml")
        html_lyrics = str(soup_song.find(attrs={'class': "lyric-body"}))
        if html_lyrics == 'None':
            continue # skips empty lyrics
        else:
            lyrics = re.sub('<.+?>+?', '', html_lyrics)
            lyrics = re.sub('\[', '', lyrics)
            lyrics = re.sub('\]', '', lyrics)
            num_len = len(str(len(lyrics_links))) # length of number of songs
            with open('./songs/' + artist_name.replace(' ', '_') + '_' + str(i).zfill(num_len) + '.txt', 'w') as f:
                #zfill(num_len) makes sure there are enough leading 0's before the number to sort files easily.
                f.writelines(lyrics)
            list_lyrics_artist(artist_name, i)
def list_lyrics_artist(artist_name, num_songs):
    '''
    Creates two lists of strings: one with the i lyrics and another with the name of the artist i times.
    '''
    artist_list = [artist_name for i in range(num_songs)]
    num_len = len(str(num_songs))
    lyrics_list = []
    for i in range(1, num_songs + 1):
        with open('./songs/' + artist_name.replace(' ', '_') + '_' + str(i).zfill(num_len) + '.txt', 'r') as f:
            lyrics_list.append(f.read())
    return lyrics_list, artist_list
def build_model(X, y):
    '''
    Trains Logistic Regression model
    '''
    Xtrain, Xtest, ytrain, ytest = tts(X, y)
    cv = CountVectorizer()
## Main Program:

X = []
y = []

lyrics_list, artist_list = list_lyrics_artist()
X.append(lyrics_list)
y.append(artist_list)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13794212218649518, 'fraction_numerical': 0.007395498392282958, 'mean_word_length': 3.38075117370892, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 7, 'lorem ipsum': 0, 'www.': 7, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '406244', 'license': 'MIT', 'n_tokens_mistral': 3192, 'n_tokens_neox': 3111, 'n_words': 827, 'path': 'Lyrics_project.ipynb', 'repo_name': 'fserro/lyrics-classifier', 'size': '9330'}