<jupyter_start><jupyter_text>MusicStack<jupyter_code>######################################################
# Juypter
######################################################
%load_ext autoreload
%autoreload
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("""<style>div.output_area{max-height:10000px;overflow:scroll;}</style>"""))


######################################################
# General
######################################################
from ioUtils import saveJoblib, loadJoblib, saveFile, getFile
import urllib
from urllib.parse import quote
from timeUtils import clock, elapsed
from fsUtils import moveFile, setFile, setDir, setSubDir, isFile, isDir, mkDir
from ioUtils import getFile, saveFile
from fileUtils import getFileBasics, getBasename
from webUtils import getWebData, getHTML, getURL
from time import sleep
import time
from multiprocessing import Pool


######################################################
# Music Stuff
######################################################
from discogsBase import discogs
from artistMS import artistMS
from discogsUtils import musicstackUtils
from multiArtist import multiartist
from myMusicDBMap import myMusicDBMap


######################################################
# Versions
######################################################
import sys
print("Python: {0}".format(sys.version))
import datetime as dt
start = dt.datetime.now()
print("Notebook Last Run Initiated: "+str(start))<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Initialize<jupyter_code>amd = artistMD()
bsdata = getHTML("/Users/tgadfort/discogs/artists-musicstack/data/Glimmer (4) Discography and Albums - MusicStack.htm")
retval = amd.getData("/Users/tgadfort/discogs/artists-musicstack/data/Glimmer (4) Discography and Albums - MusicStack.htm")
retval.show()

    
    
    ##################################################################################################################
    # Parse Downloaded Files
    ##################################################################################################################
    def parseDownloadedFiles(self):
        artistDir = self.disc.getArtistsDir()
        dataDir   = setDir(artistDir, "data")
        files     = findPatternExt(dataDir, pattern="Rate Your Music", ext=".html")
        #"/Volumes/Biggy/Discog/artists-rateyourmusic/data/", pattern="Rate Your Music", ext=".html")
        for ifile in files:
            htmldata = getFile(ifile)
            retval   = self.getData(ifile)
            artistID = retval.ID.ID
            savename = self.getArtistSavename(artistID)
            saveFile(idata=htmldata, ifile=savename, debug=True)
bsdata
%load_ext autoreload
%autoreload
discMS = discogs('musicstack')
dutils = musicstackUtils()
artMS  = artistMS()
artsMS = artistsMS(discMS)<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>My Music<jupyter_code>mdb = myMusicDBMap(debug=True)<jupyter_output>Creating myMusicDBMap()
/Users/tgadfort/Documents/code/discogs
   Loading my music db map: /anaconda3/envs/py37/musicdb/myMusicMap.p
   DB keys: ['Discogs', 'AllMusic', 'MusicBrainz', 'AceBootlegs', 'RateYourMusic', 'LastFM', 'DatPiff', 'RockCorner']
Counter({'Discogs': 3851, 'AllMusic': 3760, 'MusicBrainz': 3622, 'LastFM': 1480, 'AceBootlegs': 80, 'DatPiff': 56, 'RateYourMusic': 22, 'RockCorner': 3})<jupyter_text>Search<jupyter_code>from ioUtils import getFile, saveFile
from searchUtils import findPatternExt
from fileUtils import getBaseFilename

artMS = artistMS()
files = findPatternExt("/Users/tgadfort/Downloads/", pattern="MusicStack", ext=".html")
for ifile in files:
    fname  = getBaseFilename(ifile)
    artist = fname[:fname.find(" Vinyl Records")]
    artsMS.searchMusicStackForArtist(artist=artist, data=ifile, debug=False)<jupyter_output>Saving 4 new artist media to /Volumes/Biggy/Discog/artists-musicstack/28/4134828.p
  --> This file is 284B.
Saving 303 new artist media to /Volumes/Biggy/Discog/artists-musicstack/24/6749124.p
  --> This file is 11.8kB.
Saving 225 new artist media to /Volumes/Biggy/Discog/artists-musicstack/55/3885155.p
  --> This file is 7.8kB.<jupyter_text>Download All Music<jupyter_code>def downloadMusicStackArtists(minI=-1):
    artists = mdb.getArtists()
    for i,artistName in enumerate(artists):
        if i <= minI:
            continue
        print("====>",i,"/",len(artists),"  : ",artistName)
        artsMS.searchMusicStackForArtist(artistName, sleeptime=15)
        break
downloadMusicStackArtists(0)
for i,searchTerm in enumerate(searchTerms):
    print("\n=====>",i,"/",len(searchTerms),"==>",searchTerm)
    artsMS.searchRateYouDPusicForArtist(searchTerm)
    known.append(searchTerm)
    sleep(3)<jupyter_output><empty_output><jupyter_text>Initialize<jupyter_code>%load_ext autoreload
%autoreload
discMS = discogs('musicstack')
dutils = cdandlpUtils()
artMS  = artistMS()
artsMS = artistsMS(discMS)<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Music DB<jupyter_code>mdb = myMusicDBMap(debug=True)<jupyter_output>Creating myMusicDBMap()
/Users/tgadfort/Documents/code/discogs
   Loading my music db map: /anaconda3/envs/py37/musicdb/myMusicMap.p
   DB keys: ['Discogs', 'AllMusic', 'MusicBrainz', 'AceBootlegs', 'RateYourMusic', 'LastFM', 'DatPiff', 'RockCorner']
Counter({'Discogs': 3851, 'AllMusic': 3760, 'MusicBrainz': 3622, 'LastFM': 1480, 'AceBootlegs': 80, 'DatPiff': 56, 'RateYourMusic': 22, 'RockCorner': 3})<jupyter_text>Search<jupyter_code>#artsCL.searchCDandLPForArtist("<NAME>", data=data)<jupyter_output><empty_output><jupyter_text>Download All Music<jupyter_code>def downloadCDandLPArtists(minI=-1):
    artists = mdb.getArtists()
    for i,artistName in enumerate(artists):
        if i <= minI:
            continue
        print("====>",i,"/",len(artists),"  : ",artistName)
        artsCL.searchCDandLPForArtist(artistName, sleeptime=15)
downloadCDandLPArtists(1)
#data = getFile("Search A Flock Of Seagulls on CDandLP.html")
#artsCL.searchCDandLPForArtist("A Flock Of Seagulls", data=data)
retval = artCL.getData("/Volumes/Biggy/Discog/artists-cdandlp/65/63161947632565.p")
url="https://listen.tidal.com/artist/3565368"
savename="tidal.p"
artsMS.downloadArtistURL(url, savename)
bsdata = getFile("tidal.p")
bsdata
retval.artist.get()
dutils.getArtistID("https://www.cdandlp.com/en/a-flock-of-seagulls/artist/")
retval.url.get()
retval.ID.get()
retval.pages.get()
for k,v in retval.media.media.items():
    for v2 in v:
        print(v2.get())<jupyter_output>{'album': "(It'S Not Me) Talking (1983)", 'url': 'https://www.cdandlp.com/en/a-flock-of-seagulls/it-s-not-me-talking-1983/7inch-sp/r3446051686/', 'aclass': None, 'aformat': None, 'artist': ['A Flock Of Seagulls'], 'code': '98207858416772', 'year': None, 'err': None}
{'album': "(It'S Not Me) Talking / I Ran (Live)", 'url': 'https://www.cdandlp.com/en/a-flock-of-seagulls/it-s-not-me-talking-i-ran-live/7inch/r3100688049/', 'aclass': None, 'aformat': None, 'artist': ['A Flock Of Seagulls'], 'code': '4757513217180', 'year': None, 'err': None}
{'album': "(It'S Not Me) Talking", 'url': 'https://www.cdandlp.com/en/a-flock-of-seagulls/it-s-not-me-talking/lp/r3109042535/', 'aclass': None, 'aformat': None, 'artist': ['A Flock Of Seagulls'], 'code': '55769514039938', 'year': None, 'err': None}
{'album': 'Jive– Hop 201 - Uk - 1982', 'url': 'https://www.cdandlp.com/en/a-flock-of-seagulls-a-flock-of-seagulls/a-flock-of-seagulls-a-flock-of-seagulls/lp/r119628061/', 'aclass': None, 'aformat': None, 'ar[...]
{'fraction_non_alphanumeric': 0.21835524619747357, 'fraction_numerical': 0.030420211394689353, 'mean_word_length': 5.836123348017622, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 42, 'https://': 6, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '337447', 'license': 'MIT', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2604, 'n_words': 514, 'path': 'notebooks/old/Download Artists MusicStack.ipynb', 'repo_name': 'tgadf/dbmaster', 'size': '7761'}