<jupyter_start><jupyter_text>Scraping Lyrics Using MetroLyrics and LastFM Updated on 11/12/2019----------------  To create conda environment from file:1. `cd` into `pyGhostWriter` repo2. `conda env create --file environment.yml`3. (Optional) `source ~/.bashrc`<jupyter_code>!pip install --upgrade pip; pip install python-decouple; pip install tswift;
import string
import random

import numpy as np
import pandas as pd
import requests as r
from tswift import Artist, Song
from decouple import config

# LastFM
API_KEY = config('LASTFM_API_KEY', cast=str)<jupyter_output><empty_output><jupyter_text>MetroLyrics with `tswift` Usage Example`tswift` is currently broken, and I suspect its due to a change in the MetroLyrics API. I've submitted an issue on Github, but until that is resolved another solution needs to be implemented to retrieve lyrics for a given song or artist. **TODO**<jupyter_code># Verify tswift is operational
the_cure = Artist('The Cure')
the_cure.songs[:10]
# <|endoftext|> is the GPT-2 training data delimiter
song = random.choice(the_cure.songs)
print(song.lyrics+'\n<|endoftext|>')
# Create dataframe of all songs and lyrics for The Cure
lyrics_dict = {s.title: s.lyrics for s in the_cure.songs}
lyrics_zipped = list(zip(lyrics_dict.keys(), lyrics_dict.values()))
lyrics_df = pd.DataFrame(lyrics_zipped, columns=['title', 'lyrics'])

print('Observations/Songs: ', len(lyrics_df))
lyrics_df.head()<jupyter_output>Observations/Songs:  301<jupyter_text>LastFM Query ExamplesThe API expects UTF-8 encoded strings, but in Python 3 this is the default string encoding.<jupyter_code># Query similar artists of The Cure
# To run LastFM queries input your API key
response = r.get(f'http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist=thecure&api_key={API_KEY}&format=json')
response
resp_json = response.json()['similarartists']['artist']

print([band['name'] for band in resp_json])
# Query top tags for The Cure
response = r.get(f'http://ws.audioscrobbler.com/2.0/?method=artist.getTopTags&artist=thecure&api_key={API_KEY}&format=json')
response
resp_json = response.json()['toptags']['tag']

print([tag['name'] for tag in resp_json])
# Query top artists in the post-punk top tag
response = r.get(f'http://ws.audioscrobbler.com/2.0/?method=tag.gettopartists&tag=post-punk&api_key={API_KEY}&format=json')
response
resp_json = response.json()['topartists']['artist']

print([band['name'] for band in resp_json])<jupyter_output>['The Cure', 'Joy Division', 'Nick Cave & The Bad Seeds', 'Swans', 'Siouxsie and the Banshees', 'Echo & the Bunnymen', 'Bauhaus', 'She Wants Revenge', 'Wire', 'Parquet Courts', 'The Fall', 'Killing Joke', 'Gang of Four', 'Motorama', 'Iceage', 'Television', 'Idles', 'The Chameleons', 'Public Image Ltd.', 'Savages', 'These New Puritans', 'Les Savy Fav', 'The Soft Moon', 'Protomartyr', 'New Model Army', 'The The', 'Mission of Burma', 'Suicide', 'Felt', 'Lebanon Hanover', 'The Durutti Column', 'The Birthday Party', 'Crystal Stilts', '<NAME>', 'The Gun Club', 'The Sound', 'Orange Juice', 'The Feelies', 'Pere Ubu', 'Magazine', 'Young Marble Giants', 'Sleaford Mods', 'Ought', 'DRAB MAJESTY', 'Love And Rockets', 'Preoccupations', 'Tuxedomoon', 'The Clean', 'Буерак', 'Television Personalities']<jupyter_text>Combining LastFM and MetroLyrics<jupyter_code># Choose a band, and instantiate the Artist class for it, if it exists.
user_band = input('Input artist (ex: The Cure): ')

try:
    band = Artist(user_band)
    if len(band.songs) == 0:
        raise ValueError('No songs for this artist.')
except Exception as e:
    print(f'Exception occured: {e}')
else:
    print(f'Your artist has been chosen: {band}. Some of their songs include:')
    print('* '+'\n* '.join([song.title for song in band.songs[:10]]))
# Before collecting any lyrics, build a list of all artists similar to the chosen artist.
def query_lastfm(artist, query_type=2):
    
    artist_name = str(artist.name).replace('-', '')
    
    if query_type == 1:
        # Get similar artists
        response = r.get(f'http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={artist_name}&api_key={API_KEY}&format=json')
        if response.status_code != 200:
            raise IOError(r.status_codes._codes[response.status_code])

        resp_json = response.json()['similarartists']['artist']
        similar_artists = [similar_artist['name'] for similar_artist in resp_json]
        return similar_artists
        
    elif query_type == 2:
        # Get similar artists
        response = r.get(f'http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={artist_name}&api_key={API_KEY}&format=json')
        if response.status_code != 200:
            raise IOError(f'IOError: {r.status_codes._codes[response.status_code]}')
        resp_json = response.json()['similarartists']['artist']
        similar_artists = [similar_artist['name'] for similar_artist in resp_json]
        
        # Get top tags
        response = r.get(f'http://ws.audioscrobbler.com/2.0/?method=artist.getTopTags&artist={artist_name}&api_key={API_KEY}&format=json')
        resp_json = response.json()['toptags']['tag']
        top_tags = [tag['name'] for tag in resp_json]
        
        # Get tags top artists
        tag_top_artists = []
        for tag in top_tags[:3]:
            response = r.get(f'http://ws.audioscrobbler.com/2.0/?method=tag.gettopartists&tag={tag}&api_key={API_KEY}&format=json')
            resp_json = response.json()['topartists']['artist']
            tag_top_artists.extend([artist['name'] for artist in resp_json])
            
        return similar_artists, top_tags, tag_top_artists
          
    else:
        raise ValueError(f'query_type {query_type} unavailable.')

similar_artists, top_tags, tag_top_artists = query_lastfm(band)
print(similar_artists)
print(top_tags)
print(tag_top_artists)<jupyter_output>['P<NAME>d', 'Porcupine Tree', 'Rush', 'Coheed and Cambria', 'The Mars Volta', 'Genesis', '<NAME>', '<NAME>', 'King Crimson', 'Yes', '<NAME>', '<NAME>', 'Riverside', 'dredg', '<NAME>', 'Marillion', 'The Alan Parsons Project', 'Camel', 'The Dear Hunter', 'Blackfield', '<NAME>', 'Roger Waters', 'Emerson, Lake & Palmer', 'Antimatter', 'Karnivool', 'Oceansize', 'The Pineapple Thief', 'Gentle Giant', 'Asia', 'Fair to Midland', 'Rishloo', '<NAME>', 'Lunatic Soul', 'Ozric Tentacles', 'Gazpacho', "Spock's Beard", 'Van der Graaf Generator', 'Wishbone Ash', 'Pure Reason Revolution', 'Soft Machine', 'Closure in Moscow', 'Nothing More', 'Steve Hackett', 'Focus', 'Kingston Wall', 'Amplifier', 'Gong', '<NAME>', 'Caravan', 'Eloy', 'Animal Collective', 'Portugal. The Man', 'Buckethead', 'Chelsea Wolfe', 'Panda Bear', 'Dirty Projectors', 'Bibio', '<NAME>', 'Deerhoof', 'Captain Beefheart & His Magic Band', 'Health', 'Liars', 'Shlohmo', 'The Books', 't[...]
{'fraction_non_alphanumeric': 0.15177263404629357, 'fraction_numerical': 0.006738939349545854, 'mean_word_length': 4.941688424717145, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '585921', 'license': 'MIT', 'n_tokens_mistral': 2375, 'n_tokens_neox': 2274, 'n_words': 614, 'path': 'Lyrics_Scraping.ipynb', 'repo_name': 'beverast/pyGhostWriter', 'size': '6885'}