<jupyter_start><jupyter_text>The Rocker Corner<jupyter_code>######################################################
# Juypter
######################################################
%load_ext autoreload
%autoreload
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("""<style>div.output_area{max-height:10000px;overflow:scroll;}</style>"""))


######################################################
# General
######################################################
from ioUtils import saveJoblib, loadJoblib, saveFile, getFile
import urllib
from urllib.parse import quote
from collection import collections
from artist import artist
from timeUtils import clock, elapsed
from fsUtils import moveFile, setFile, setDir, setSubDir, isFile, isDir, mkDir
from ioUtils import getFile, saveFile
from fileUtils import getFileBasics, getBasename
from webUtils import getWebData, getHTML, getURL
from time import sleep
import time
from multiprocessing import Pool


######################################################
# Music Stuff
######################################################
from discogsBase import discogs
from artistRC import artistRC
from artistsRC import artistsRC
from discogsUtils import rockcornerUtils
from multiArtist import multiartist
from myMusicDBMap import myMusicDBMap


######################################################
# Versions
######################################################
import sys
print("Python: {0}".format(sys.version))
import datetime as dt
start = dt.datetime.now()
print("Notebook Last Run Initiated: "+str(start))<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Initialize<jupyter_code>%load_ext autoreload
%autoreload
discRC = discogs('rockcorner')
dutils = rockcornerUtils()
artRC  = artistRC()
artsRC = artistsRC(discRC)<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
Creating /Users/tgadfort/Documents/code/discogs/artists-rockcorner
Creating /Users/tgadfort/Documents/code/discogs/artists-rockcorner<jupyter_text>Music DB<jupyter_code>mdb = myMusicDBMap(debug=True)<jupyter_output>Creating myMusicDBMap()
/Users/tgadfort/Documents/code/discogs
   Loading my music db map: /anaconda3/envs/py37/musicdb/myMusicMap.p
   DB keys: ['Discogs', 'AllMusic', 'MusicBrainz', 'AceBootlegs', 'RateYourMusic', 'LastFM', 'DatPiff', 'RockCorner']
Counter({'Discogs': 3851, 'AllMusic': 3760, 'MusicBrainz': 3622, 'LastFM': 1480, 'AceBootlegs': 80, 'DatPiff': 56, 'RateYourMusic': 22, 'RockCorner': 3})<jupyter_text>Test<jupyter_code>url="https://www.therockcorner.com/alannah-myles/"
savename="trc.p"
from webUtils import getWebData
getWebData(base=url, savename=savename, useSafari=False)
bsdata = getHTML("trc.p")<jupyter_output><empty_output><jupyter_text>Search<jupyter_code>artsRC.searchRockCornerForArtist("Alannah Myles")<jupyter_output>===================== Searching For Alannah Myles =====================
Found 4 artists
1 / 4 	: 341093961 	 https://www.therockcorner.com/alannah-myles
Saving /Users/tgadfort/Music/Discog/artists-rockcorner/61/341093961.p (force=False)
  --> This file is 13.8kB.
Done. Sleeping for 2 seconds
2 / 4 	: 307006520 	 https://www.therockcorner.com/sevendust
Saving /Users/tgadfort/Music/Discog/artists-rockcorner/20/307006520.p (force=False)
  --> This file is 14.4kB.
Done. Sleeping for 2 seconds
3 / 4 	: 718959345 	 https://www.therockcorner.com/disturbed
Saving /Users/tgadfort/Music/Discog/artists-rockcorner/45/718959345.p (force=False)
  --> This file is 13.9kB.
Done. Sleeping for 2 seconds
4 / 4 	: 379499196 	 https://www.therockcorner.com/slash
Saving /Users/tgadfort/Music/Discog/artists-rockcorner/96/379499196.p (force=False)
  --> This file is 14.4kB.
Done. Sleeping for 2 seconds<jupyter_text>Download All Music<jupyter_code>def downloadRockerCornerArtists(minI=-1):
    artists = mdb.getArtists()
    for i,artistName in enumerate(artists):
        if i <= minI:
            continue
        print("====>",i,"/",len(artists),"  : ",artistName)
        artsRC.searchRockCornerForArtist(artistName, sleeptime=30)
        sleep(30)
downloadRockerCornerArtists(1156)
def getMyMusicDBMap():
    myMusicMap = getFile(ifile="/Users/tgadfort/Documents/code/music/myMusicMap.p")
    print("Found {0} entries".format(len(myMusicMap)))
    return myMusicMap

myMusicMap = getMyMusicDBMap()
def downloadSearchResults(maxI=-1):
    for i,artistName in enumerate(myMusicMap.keys()):
        if i <= maxI:
            continue
        print("====>",i,"/",len(myMusicMap),"  : ",artistName)
#        try:
        artsLM.searchLastFMForArtist(artistName, sleeptime=3)
#        except:
#            sleep(3)
        sleep(3)
downloadSearchResults(3544)<jupyter_output><empty_output><jupyter_text>Download Extra Artist Files<jupyter_code>#artsLM.assertDBModValExtraData(1, maxPages=3)
from random import shuffle
modVals = list(range(100))
shuffle(modVals)
for modVal in modVals:
    try:
        artsLM.assertDBModValExtraData(modVal, maxPages=3)
    except:
        continue
artsLM.assertDBModValExtraData(61)
retval = artLM.getData("/Volumes/Biggy/Discog/artists-lastfm/51/34806831451.p")
artsLM.parseArtistFiles()
dutils.getArtistID("Coldplay vs. The Killers vs. <NAME> vs. <NAME> vs. Empire Of The Sun vs. Muse")  # 99596595167
dutils.getArtistID("Coldplay vs. The Killers vs. <NAME> vs. Bon…")   # 93496498096
url="https://www.last.fm/search/artists?q=Coldplay"
savename="lm_search.p"
artsLM.downloadArtistURL(url=url, savename=savename)
bsdata = getHTML("lm_search.p")
uls = bsdata.findAll("ul", {"class": "artist-results"})
for ul in uls:
    lis = ul.findAll("li", {"class": "artist-result"})
    for li in lis:
        h4 = li.find("h4")
        if h4 is None:
            raise ValueError("No h4 in list")
        result   = artLM.getNamesAndURLs(h4)
        name     = result[0].name
        url      = result[0].url
        artistID = dutils.getArtistID(name)
        print(name,'\t',url,'\t',artistID)
#bsdata
retval = artLM.getData("lastfm.p")
retval.artist.get()
retval.pages.get()
retval.url.get()
retval.ID.get()
retval.profile.get()
for album in retval.media.media["Albums"]:
    print(album.get())
retval.mediaCounts.get()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19054373522458629, 'fraction_numerical': 0.032781717888100866, 'mean_word_length': 5.436105476673428, 'pattern_counts': {'":': 2, '<': 34, '<?xml version=': 0, '>': 38, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '159363', 'license': 'MIT', 'n_tokens_mistral': 2312, 'n_tokens_neox': 2089, 'n_words': 430, 'path': 'notebooks/old/Download Artists TheRockerCorner.ipynb', 'repo_name': 'tgadf/dbmaster', 'size': '6363'}