<jupyter_start><jupyter_text>PART 1<jupyter_code>import requests
from bs4 import BeautifulSoup
import re
import pandas as pd

response = requests.get('https://www.azlyrics.com/r/rem.html')
doc = BeautifulSoup(response.text)
songs = doc.find(id = 'listAlbum').find_all('a')
rows = []
for song in songs:
    row = {}
    row['title'] = song.text.strip()
    row['url'] = song['href'].strip()
    rows.append(row)
df = pd.DataFrame(rows)
df['url'] = df.url.str.extract('.+(/lyrics.+)')
df.head()
df.to_csv('AZLyrics_REM.csv', index = False)<jupyter_output><empty_output><jupyter_text>Part 2: Fields to scrape<jupyter_code># Then, open your search results csv, and scrape the following field:

# Lyrics
# Merge with your original song information and save as a new CSV file
df = pd.read_csv('AZLyrics_REM.csv')
def get_lyrics(song):
    row = {}
    headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
    url = 'https://www.azlyrics.com/' + song['url']
    response = requests.get(url, headers = headers)
    doc = BeautifulSoup(response.text)
#     print(doc.find('div', class_ = 'ringtone'))
    row['title'] = doc.find('div', class_ = 'ringtone').find_next_sibling('b').text.strip()
    row['lyrics'] = lyrics = doc.find('div', class_ = 'ringtone').find_next_sibling('div').text.strip()
    return pd.Series(row)
df = df.apply(get_lyrics, axis = 1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16891891891891891, 'fraction_numerical': 0.02027027027027027, 'mean_word_length': 4.6311787072243344, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '168250', 'license': 'MIT', 'n_tokens_mistral': 574, 'n_tokens_neox': 538, 'n_words': 115, 'path': '09-homework/AZLyrics.ipynb', 'repo_name': 'talola612/foundations_homework', 'size': '1480'}