<jupyter_start><jupyter_text>Global<jupyter_code>## Basic stuff
%load_ext autoreload
%autoreload
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("""<style>div.output_area{max-height:10000px;overflow:scroll;}</style>"""))

## Python Version
import sys
print("Python: {0}".format(sys.version))


################################################################################
## General Stuff
################################################################################
from ioUtils import saveJoblib, loadJoblib, saveFile, getFile
from timeUtils import clock, elapsed
from fsUtils import moveFile, setFile, setDir, setSubDir, isFile, isDir, mkDir
from fileUtils import getFileBasics, getBasename
from webUtils import getHTML
from multiprocessing import Pool


######################################################
# Music Stuff
######################################################
from dbBase import dbBase
from multiArtist import multiartist
from myMusicDBMap import myMusicDBMap
from mainDB import mainDB
from parseDBArtistsData import parseDBArtistsData




import datetime as dt
start = dt.datetime.now()
print("Notebook Last Run Initiated: "+str(start))<jupyter_output><empty_output><jupyter_text>Deezer<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsDeezer import dbArtistsDeezer
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertExtra, dbArtistsUnofficial, dbArtistsAssertUnofficial
from time import sleep

for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsDeezer())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
        
sleep(1)
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("Deezer")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Genius<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsGenius import dbArtistsGenius
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertExtra, dbArtistsUnofficial, dbArtistsAssertUnofficial
from time import sleep

for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsGenius())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
        
sleep(1)
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("Genius")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>AlbumOfTheYear<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsAlbumOfTheYear import dbArtistsAlbumOfTheYear
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertCredit
from time import sleep

for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsAlbumOfTheYear())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)

sleep(1)
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("AlbumOfTheYear")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>AllMusic<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsAllMusic import dbArtistsAllMusic
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertCredit
# This downloads missing credit files
for modVal in range(100):
    try:
        dbac = dbArtistsAssertCredit(dbArtistsAllMusic())
        dbac.parseModVal(modVal)
        dbac.downloadUnknownArtistCredits()
    except:
        continue

%load_ext autoreload
%autoreload
from dbArtistsAllMusic import dbArtistsAllMusic
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertCredit
from time import sleep

for modVal in range(100):
    break
    dbAP = dbArtistsPrimary(dbArtistsAllMusic())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
        
        
# This downloads missing credit files
for modVal in range(100):
    try:
        dbac = dbArtistsAssertCredit(dbArtistsAllMusic())
        dbac.parseModVal(modVal)
        dbac.downloadUnknownArtistCredits()
    except:
        continue

for modVal in range(100):
    dbAP = dbArtistsCredit(dbArtistsAllMusic())
    dbAP.parse(modVal, previousDays=200)
    

for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsAllMusic())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa or True:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
    
    
sleep(1)
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("AllMusic")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>LastFM<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsLastFM import dbArtistsLastFM
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertCredit
from time import sleep

for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsLastFM())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)

sleep(1)
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("LastFM")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>MusicBrainz<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsMusicBrainz import dbArtistsMusicBrainz
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertCredit
from time import sleep

start, cmt = clock("Creating DB")
for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsMusicBrainz())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)

sleep(1)
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("MusicBrainz")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Discogs<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsDiscogs import dbArtistsDiscogs
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertExtra, dbArtistsUnofficial, dbArtistsAssertUnofficial
from time import sleep

start, cmt = clock("Creating DB")
for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsDiscogs())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
        
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("Discogs")
elapsed(start, cmt)<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
Current Time is Tue Mar 02, 2021 13:46:32 for Creating DB
=========================   discogs   =========================
  ===> Found [New=0] / [Rec=15] / [Tot=15] files.
  ===> Found new 15 files (< None days) to parse for modval 0
  ===> Found 7776 previous data for ModVal=0
Saving 15 new artist IDs to /Users/tgadfort/Music/Discog/artists-discogs-db/0-DB.p
Saving 7791 total artist IDs to /Users/tgadfort/Music/Discog/artists-discogs-db/0-DB.p
Saving 73459 total artist media
  --> This file is 11.7MB.
  ===> Found 7791 previous data for ModVal=0
Saving 7791 new artist IDs name data to /Users/tgadfort/Music/Discog/artists-discogs-db/metadata/0-Metadata.p
  --> This file is 450.2kB.
  ===> Found 7791 previous data for ModVal=0
Saving 7791 new artist IDs name data to /Users/tgadfort/Music/Discog/artists-discogs-db/metadata/0-MediaMetadata.p
  --> This file is 4.2MB.
=========================   discogs[...]<jupyter_text>RateYourMusic<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsRateYourMusic import dbArtistsRateYourMusic
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertCredit, dbArtistsRawHTML

start, cmt = clock("Creating DB")
dbAP = dbArtistsRawHTML(dbArtistsRateYourMusic())
dbAP.parse(previousDays=1)

for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsRateYourMusic())
    dometa = dbAP.parse(modVal, previousDays=None, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
        
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("RateYourMusic")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>KWorbSpotify<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsKWorbSpotify import dbArtistsKWorbSpotify
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertExtra, dbArtistsUnofficial, dbArtistsAssertUnofficial, dbArtistsRawFiles
from time import sleep

dbAP = dbArtistsRawFiles(dbArtistsKWorbSpotify(), "data")
#dbAP.parse(previousDays=1000000)

for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsKWorbSpotify())
    dometa = dbAP.parse(modVal, previousDays=3, force=False)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
        
sleep(1)
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("KWorbSpotify")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>KWorbiTunes<jupyter_code>dbAP = dbArtistsPrimary(dbArtistsKWorbiTunes())
dometa = dbAP.parse(0, previousDays=3, force=True)
if dometa:
    dbAP.createArtistMetadata(modVal)
    dbAP.createAlbumMetadata(modVal)
%load_ext autoreload
%autoreload
from dbArtistsKWorbiTunes import dbArtistsKWorbiTunes
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertExtra, dbArtistsUnofficial, dbArtistsAssertUnofficial, dbArtistsRawFiles
from time import sleep

dbAP = dbArtistsRawFiles(dbArtistsKWorbiTunes(), "data")
#dbAP.parse(previousDays=1000000)

for modVal in range(1):
    continue
    dbAP = dbArtistsPrimary(dbArtistsKWorbiTunes())
    dometa = dbAP.parse(modVal, previousDays=3, force=True)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
        
sleep(1)
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("KWorbiTunes")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>KWorbYouTube<jupyter_code>%load_ext autoreload
%autoreload
from dbArtistsKWorbYouTube import dbArtistsKWorbYouTube
from dbArtistsParse import dbArtistsPrimary, dbArtistsCredit, dbArtistsExtra
from dbArtistsParse import dbArtistsAssertExtra, dbArtistsUnofficial, dbArtistsAssertUnofficial, dbArtistsRawFiles
from time import sleep

dbAP = dbArtistsRawFiles(dbArtistsKWorbYouTube(), "data")
#dbAP.parse(previousDays=1000000)

for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsKWorbYouTube())
    dometa = dbAP.parse(modVal, previousDays=3, force=True)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
        
sleep(1)
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("KWorbYouTube")
elapsed(start, cmt)    
dbAP = dbArtistsPrimary(dbArtistsKWorbYouTube())
dbAP.parse(0, previousDays=3, force=False)
retval = dbArtistsKWorbYouTube().artist.getData("/Volumes/Piggy/Discog/artists-kworbyoutube/0/16247211200.p")
retval.show()


for modVal in range(100):
    dbAP = dbArtistsPrimary(dbArtistsKWorbYouTube())
    dometa = dbAP.parse(modVal, previousDays=3, force=True)
    if dometa:
        dbAP.createArtistMetadata(modVal)
        dbAP.createAlbumMetadata(modVal)
%autoreload
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("KWorbYouTube")
elapsed(start, cmt)    
%autoreload
from webUtils import getHTML
from searchUtils import findExt
dbAP = dbArtistsRawFiles(dbArtistsKWorb(), "spotify")
dbAP.parse(previousDays=10)<jupyter_output><empty_output><jupyter_text>LastFM<jupyter_code>start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("LastFM")
elapsed(start, cmt)    
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("AlbumOfTheYear")
elapsed(start, cmt)    
start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull("Genius")
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Create Metadata (Only)<jupyter_code>dbs    = ['AllMusic', 'Discogs', 'MusicBrainz', 'LastFM']
dbs    = ['AllMusic']
dbs    = []
if len(dbs) > 0:
    start,cmt=clock("Parsing Metadata")
    for db in dbs:
        pdb.parseMetadata(db=db, nProcs=3, modVals=list(range(100)), force=True) 
    elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Parse Downloads<jupyter_code>dbs = ["RateYourMusic", "MusicStack"]
dbs = ["RateYourMusic"]
if len(dbs) > 0:
    start,cmt=clock("Parsing Downloads")
    for db in dbs:
        pdb.parseDownloads(db)
    elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Parse Everything<jupyter_code>%load_ext autoreload
%autoreload
from mainDB import mainDB
from parseDBArtistsData import parseDBArtistsData
maindb = mainDB(mdb=None, create=True, debug=False)
pdb    = parseDBArtistsData(maindb=maindb)


dbs    = ['AllMusic', 'Discogs', 'MusicBrainz', 'LastFM']
#dbs = ['MusicBrainz', 'LastFM']
dbs = ["Discogs"]
#dbs = ["Discogs", "MusicBrainz"]
dbs = ["AllMusic"]
#dbs = ["RateYourMusic"]
#dbs = ["MusicBrainz"]
#dbs = []
start,cmt=clock("Parsing Everything")
for db in dbs:
    s = 10
    #pdb.parse(db, nProcs=3, modVals=list(range(s,s+9)), force=False)
    #pdb.parse(db, nProcs=4, modVals=list(range(10,100)), force=True)
    #pdb.parse(db, nProcs=3, modVals=list(range(100)), force=False, extra=False)
    #pdb.parse(db, nProcs=3, modVals=[88,89,87,37,98,97], force=True, extra=False)
    #pdb.parse(db, nProcs=1, modVals=[9], force=True)
    pdb.parse(db, nProcs=3, modVals=list(range(100)), force=False, primary=True)
    #pdb.parse(db, nProcs=2, modVals=list(range(100)), force=False)
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Parse Extra Data<jupyter_code>dbs = ['AllMusic', 'Discogs', 'MusicBrainz', 'LastFM']
dbs = ['MusicBrainz']

%load_ext autoreload
%autoreload
from parseDBArtistsData import parseDBArtistsData
pdb    = parseDBArtistsData(maindb=maindb)

start,cmt=clock("Parsing Everything")
for db in dbs:
    #pdb.parse(db, nProcs=3, modVals=list(range(100)), force=False, extra=True)
    #pdb.parse(db, nProcs=1, modVals=list(range(80,100)), force=False, extra=True)
    #pdb.parse(db, nProcs=1, modVals=list(range(80)), force=False, extra=True)
    pdb.parse(db, nProcs=1, modVals=[79], force=False, extra=True)
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Parse Credit Data<jupyter_code>dbs = ['AllMusic']
#dbs = []

%load_ext autoreload
%autoreload
from parseDBArtistsData import parseDBArtistsData
pdb    = parseDBArtistsData(maindb=maindb)

start,cmt=clock("Parsing Everything")
for db in dbs:
    #pdb.parse(db, nProcs=1, modVals=[9], force=False, credit=True)
    pdb.parse(db, nProcs=3, modVals=list(range(100)), force=False, credit=True)
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>LastFM<jupyter_code>%load_ext autoreload
%autoreload
from mainDB import mainDB
from parseDBArtistsData import parseDBArtistsData

def parseLastFM(modVals=list(range(100)), force=False, primary=False, extra=False, fulldb=False):
    db = "LastFM"
    if isinstance(modVals, int):
        modVals = [modVals]
    nProcs = min(3, len(modVals))

    start,cmt=clock("Parsing Everything for {0}".format(db))
    if primary is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, force=force, primary=True)  ## Primary

    if extra is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, force=force, extra=True)  ## Extra
    
    if fulldb is True:
        maindb = mainDB(mdb=None, create=True, debug=True)
        maindb.setDBFull(db)
        elapsed(start, cmt)    
    
#parseLastFM(primary=True, fulldb=True)
parseLastFM(primary=False, fulldb=True)<jupyter_output><empty_output><jupyter_text>MusicBrainz<jupyter_code>%load_ext autoreload
%autoreload
from mainDB import mainDB
from parseDBArtistsData import parseDBArtistsData

def parseMusicBrainz(modVals=list(range(100)), force=False, primary=False, credit=False, extra=False, fulldb=False):
    db = "MusicBrainz"
    previousDays=4
    if isinstance(modVals, int):
        modVals = [modVals]
    nProcs = min(3, len(modVals))

    start,cmt=clock("Parsing Everything for {0}".format(db))
    if primary is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, previousDays=previousDays, force=force, primary=True)  ## Primary

    if extra is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, force=force, extra=True)  ## Extra
    

    if fulldb is True:
        maindb = mainDB(mdb=None, create=True, debug=True)
        maindb.setDBFull(db)
        elapsed(start, cmt)    
    
parseMusicBrainz(primary=True, force=True)
data=getFile("/Users/tgadfort/Music/Discog/artists-musicbrainz-db/0-DB.p")
data['100442956818207492535169252608810699800'].profile.external
from dbArtistsGenius import dbArtistsGenius
from time import sleep
urls=['https://genius.com/artists/Fy', 'https://genius.com/artists/Snik', 'https://genius.com/artists/Rap-viet',
     'https://genius.com/artists/Mad-clip', 'https://genius.com/artists/Moira-dela-torre', 'https://genius.com/artists/Lala-hsu',
     'https://genius.com/artists/Sin-boy', 'https://genius.com/artists/My-tam', 'https://genius.com/artists/Eladio-carrion',
     'https://genius.com/artists/Son-tung-m-tp', 'https://genius.com/artists/The-toys', 'https://genius.com/artists/The-toys-thai-artist',
     'https://genius.com/artists/Eric-chou', 'https://genius.com/artists/Dear-jane', 'https://genius.com/artists/Illeoo',
     'https://genius.com/artists/Junoflo', 'https://genius.com/artists/En', 'https://genius.com/artists/Hebe-tien',
     'https://genius.com/artists/Fer-palacio', 'https://genius.com/artists/Payung-teduh', 'https://genius.com/artists/Ysy-a',
     'https://genius.com/artists/Miss-ko', 'https://genius.com/artists/Dj-alex-arg', 'https://genius.com/artists/Bnk48',
     'https://genius.com/artists/Vu-cat-tuong', 'https://genius.com/artists/Accusefive', 'https://genius.com/artists/Mj116',
     'https://genius.com/artists/Nine-chen', 'https://genius.com/artists/Getsunova', 'https://genius.com/artists/Earth-patravee',
     'https://genius.com/artists/Alien-huang', 'https://genius.com/artists/Endy-chow', 'https://genius.com/artists/Singto-numchok',
     'https://genius.com/artists/Zi-stefan-chen', 'https://genius.com/artists/Oat-pramote', 'https://genius.com/artists/Ngot',
     'https://genius.com/artists/Bird-thongchai', 'https://genius.com/artists/Leowang', 'https://genius.com/artists/Kelly-yu-wen-wen',
     'https://genius.com/artists/Noo-phuoc-thinh', 'https://genius.com/artists/Zom-marie', 'https://genius.com/artists/Og-anic',
     'https://genius.com/artists/Maja-salvador-and-tor-saksit', 'https://genius.com/artists/Meyou', 'https://genius.com/artists/Cocktail']

urls=['https://genius.com/artists/Diskoria', 'https://genius.com/DJ_OOPs', 'https://genius.com/artists/Tugba-yurt',
     'https://genius.com/artists/Lost-sky', 'https://genius.com/artists/Dawn-oberg', 'https://genius.com/artists/Cengiz-ates',
     'https://genius.com/artists/Gal-adam', 'https://genius.com/artists/Bulat-nurimov', 'https://genius.com/artists/Taypan',
     'https://genius.com/artists/Jay-fendi-prince-of-lawtown', 'https://genius.com/artists/Ahmet-safak',
     'https://genius.com/artists/Hipsumhaps', 'https://genius.com/artists/Orri', 'https://genius.com/artists/Wonayd',
     'https://genius.com/artists/Vremya-i-steklo', 'https://genius.com/artists/Fadel-chaker', 'https://genius.com/artists/I61',
     'https://genius.com/artists/Matti-rssland', 'https://genius.com/artists/Vtornik', 'https://genius.com/artists/Helgi-bjornsson',
     'https://genius.com/artists/Herra-ylppo', 'https://genius.com/artists/Sebastian-wallden', 'https://genius.com/artists/Joker-xue',
     'https://genius.com/artists/Rkm-and-ken-y', 'https://genius.com/artists/John-mcdermott', 'https://genius.com/artists/Ronan-tynan',
     'https://genius.com/artists/Timmy-xu', 'https://genius.com/artists/John-p-kee', 'https://genius.com/artists/John-p-kee-and-the-new-life-community-choir',
     'https://genius.com/artists/Ilegales', 'https://genius.com/artists/James-wilson', 'https://genius.com/artists/Millie-b',
     'https://genius.com/artists/Stephen-paul-taylor', 'https://genius.com/artists/Byrne-and-kelly', 'https://genius.com/artists/Consequence',
     'https://genius.com/artists/Xtreme', 'https://genius.com/artists/Hua-chenyu', 'https://genius.com/artists/Bob-james',
     'https://genius.com/artists/Niska', 'https://genius.com/artists/Loon', 'https://genius.com/artists/Morgenshtern-and-eldzhey',
     'https://genius.com/artists/Morgenshtern', 'https://genius.com/artists/Jessie-morales', 'https://genius.com/artists/Jessie-morales-el-original-de-la-sierra']
urls=['https://genius.com/artists/Casino', 'https://genius.com/artists/Mark-harris', 'https://genius.com/artists/Schoolboy-q',
     'https://genius.com/artists/Hollis', 'https://genius.com/artists/Skipper', 'https://genius.com/artists/Trillville',
     'https://genius.com/artists/Cutty', 'https://genius.com/artists/Paul-mccoy']
urls=['https://genius.com/artists/Casino', 'https://genius.com/artists/Mark-harris', 'https://genius.com/artists/Schoolboy-q',
     'https://genius.com/artists/Hollis', 'https://genius.com/artists/Skipper', 'https://genius.com/artists/Trillville',
     'https://genius.com/artists/Cutty', 'https://genius.com/artists/Paul-mccoy']


for url in urls:
    print(url)
    dp = dbArtistsGenius()
    dp.downloadArtistFromURL(url)
    sleep(2)<jupyter_output><empty_output><jupyter_text>Discogs<jupyter_code>%load_ext autoreload
%autoreload
from mainDB import mainDB
from parseDBArtistsData import parseDBArtistsData

def parseDiscogs(modVals=list(range(100)), force=False, primary=False, credit=False, extra=False, fulldb=False):
    db = "Discogs"
    previousDays=5
    if isinstance(modVals, int):
        modVals = [modVals]
    nProcs = min(3, len(modVals))

    start,cmt=clock("Parsing Everything for {0}".format(db))
    if primary is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, previousDays=previousDays, force=force, primary=True)  ## Primary

    if credit is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, force=force, credit=True)  ## Credit

    if extra is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, force=force, extra=True)  ## Extra
    

    if fulldb is True:
        maindb = mainDB(mdb=None, create=True, debug=True)
        maindb.setDBFull(db)
        elapsed(start, cmt)    
    
parseDiscogs(primary=True, fulldb=True)<jupyter_output><empty_output><jupyter_text>AllMusic<jupyter_code>%load_ext autoreload
%autoreload
from mainDB import mainDB
from parseDBArtistsData import parseDBArtistsData

maindb = mainDB(mdb=None, create=True, debug=False)
pdb    = parseDBArtistsData(maindb=maindb)
pdb.parse("AllMusic", nProcs=1, modVals=[69], previousDays=100, force=True, primary=True)  ## Primary
%load_ext autoreload
%autoreload
from mainDB import mainDB
from parseDBArtistsData import parseDBArtistsData

def parseAllMusic(modVals=list(range(100)), force=False, primary=False, credit=False, fulldb=False):
    db = "AllMusic"
    previousDays=4
    if isinstance(modVals, int):
        modVals = [modVals]
    nProcs = min(3, len(modVals))

    start,cmt=clock("Parsing Everything for {0}".format(db))
    if primary is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, previousDays=previousDays, force=force, primary=True)  ## Primary

    if credit is True:
        maindb = mainDB(mdb=None, create=True, debug=False)
        pdb    = parseDBArtistsData(maindb=maindb)
        pdb.parse(db, nProcs=nProcs, modVals=modVals, force=force, credit=True)  ## Credit
    

    if fulldb is True:
        maindb = mainDB(mdb=None, create=True, debug=True)
        maindb.setDBFull(db)
        elapsed(start, cmt)    
    
#parseAllMusic(primary=True)
%load_ext autoreload
%autoreload
#parseAllMusic(credit=True)
parseAllMusic(fulldb=True)
modVal = 0
artistDBDir = dbdata["AllMusic"]["Disc"].getArtistsDBDir()
moddbdata   = dbdata["AllMusic"]["Disc"].getArtistsDBModValData(modVal)
nerrs   = 0
len(moddbdata)
from fileUtils import getBaseFilename
files = dbdata["AllMusic"]["Artists"].getArtistModValFiles(modVal, force=True)
artistIDs = {getBaseFilename(ifile): ifile for ifile in files}
toget = []
for artistID,ifile in artistIDs.items():
    if moddbdata.get(artistID) is None:
        info   = dbdata["AllMusic"]["Artists"].getData(ifile)
        artistName = info.profile.search
        if artistName is not None and len(artistName) > 0:
            info.show()
            break
        #self.creditToDownload[artistID] = [artistName,self.getArtistSavename(artistID, credit=True)]<jupyter_output><empty_output><jupyter_text>****************  Recreate Artist Names<jupyter_code>start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBArtists(recreate=False)
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Master Work<jupyter_code>start, cmt = clock("Creating DB")
maindb = mainDB(mdb=None, create=True, debug=True)
maindb.setDBFull()
elapsed(start, cmt)
df = getFile("/Users/tgadfort/Music/Discog/db-rockcorner/MasterSlimArtistDB.p")
df["MultiStatus"].value_counts()<jupyter_output><empty_output><jupyter_text>Parse Single ModVal<jupyter_code>for i in range(10):
    for j in range(10):
        print("{0}touch /Volumes/Piggy/Discog/artists-discogs/{1}/0*0*0*".format("!", i*10+j))
        print("{0}echo '{1}'".format("!", i+10*j))
masterDBDF = maindb.dbdata["MusicBrainz"]["Disc"].getMasterSlimArtistDiscogsDB()
from searchUtils import findExt
from fsUtils import setDir, setFile
modVal = 99
artistDBDir = maindb.dbdata["Discogs"]["Artists"].getArtistsDBDir()
artistDir   = maindb.dbdata["Discogs"]["Artists"].getArtistsDir()
dirVal = setDir(artistDir, str(modVal))
files  = findExt(dirVal, ext='.p')
dbname = setFile(artistDBDir, "{0}-DB.p".format(modVal))
len(files)
from datetime import datetime, timedelta    
from os import path

def getFiles():
    from searchUtils import findExt
    from fsUtils import setDir, setFile
    modVal = 99
    artistDBDir = maindb.dbdata["Discogs"]["Artists"].getArtistsDBDir()
    artistDir   = maindb.dbdata["Discogs"]["Artists"].getArtistsDir()
    dirVal = setDir(artistDir, str(modVal))
    files  = findExt(dirVal, ext='.p')
    return files

def getNewFiles(dbname, previousDays=5, force=False):
    now    = datetime.now()
    files  = getFiles()
    if isFile(dbname):
        lastModified = datetime.fromtimestamp(path.getmtime(dbname))
        if force is True:
            lastModified = None
    else:
        lastModified = None
 
    if lastModified is None:
        newFiles = files
    else:
        numNew    = [ifile for ifile in files if (now-datetime.fromtimestamp(path.getmtime(ifile))).days < previousDays]
        numRecent = [ifile for ifile in files if datetime.fromtimestamp(path.getmtime(ifile)) > lastModified]
        newFiles  = list(set(numNew).union(set(numRecent)))
        print(len(files))
        print(len(numNew))
        print(len(numRecent))
    print(len(newFiles))
artistDBDir = maindb.dbdata["Discogs"]["Artists"].getArtistsDBDir()
artistDir   = maindb.dbdata["Discogs"]["Artists"].getArtistsDir()
dirVal = setDir(artistDir, str(modVal))
dbname = setFile(artistDBDir, "{0}-DB.p".format(modVal))

files = getNewFiles(dbname=dbname, previousDays=50, force=True)
now    = datetime.now()


def parseArtistModValFiles(self, modVal, force=False, debug=False, doExtra=False):        
    from os import path
    from datetime import datetime, timedelta

    print("-"*100)
    print("Parsing Artist Files For ModVal {0}".format(modVal))
    artistInfo = self.artist

    artistDir = self.disc.getArtistsDir()
    maxModVal = self.disc.getMaxModVal()

    artistDBDir = self.disc.getArtistsDBDir()        

    dirVal = setDir(artistDir, str(modVal))
    files  = findExt(dirVal, ext='.p')
    dbname = setFile(artistDBDir, "{0}-DB.p".format(modVal))



    ### Check for recent files
    if isFile(dbname):
        lastModified = datetime.fromtimestamp(path.getmtime(dbname))
        now    = datetime.now()            
        if force is False:
            numRecent = [ifile for ifile in files if datetime.fromtimestamp(path.getmtime(ifile)) > lastModified]
            numNew    = [ifile for ifile in files if (now-datetime.fromtimestamp(path.getmtime(ifile))).days < 1]
            if len(numRecent) == 0:
                print("  ===> Found {0} files, but there are no new files to parse so skipping.".format(len(files)))
                return 0
            else:
                print("  ===> Found {0} files. There are {1} new files to parse.".format(len(files), len(numRecent)))
    else:
        force = True
        lastModified = datetime.fromtimestamp(mktime(gmtime(0)))


    if force is False:
        dbdata = getFile(dbname, version=3)
    else:
        print("Forcing Reloads of ModVal={0}".format(modVal))
        print("  Processing {0} files.".format(len(files)))
        dbdata = {}

    saveIt = 0
    for j,ifile in enumerate(files):
        if force is True:
            if j % 500 == 0:
                print("\tProcessed {0}/{1} files.".format(j,len(files)))
        artistID = getBaseFilename(ifile)
        isKnown  = dbdata.get(artistID)
        recent   = datetime.fromtimestamp(path.getctime(ifile))
        if isKnown is None or recent > lastModified or force is True:
            info   = artistInfo.getData(ifile)

            if info.ID.ID != artistID:

                # Check Profile
                try:
                    artistName = info.profile.search
                    if artistName is not None:
                        self.creditToDownload[artistID] = [artistName,self.getArtistSavename(artistID, credit=True)]
                except:
                    pass

                if debug is False:
                    continue
                print("ID From Name: {0}".format(artistID))
                print("ID From File: {0}".format(info.ID.ID))

                print("File: {0}".format(ifile))
                print("Info: {0}".format(info.url.get()))
                continue
                #1/0

            saveIt += 1
            dbdata[artistID] = info


    forceSave = False
    if saveIt > 0 and doExtra is True:
        print("\tCalling Extra Parsing")
        dbdata = self.parseArtistModValExtraFiles(modVal, dbdata=dbdata, force=force, debug=debug)
        forceSave = True
        saveIt = len(dbdata)


    if len(self.creditToDownload) > 0:
        print("Found {0} artists that need new downloads.".format(len(self.creditToDownload)))


    if saveIt > 0 or forceSave is True:
        savename = setFile(artistDBDir, "{0}-DB.p".format(modVal))     
        print("Saving {0} new artist IDs to {1}".format(saveIt, savename))
        dbNumAlbums = sum([self.getArtistNumAlbums(artistData) for artistData in dbdata.values()])
        print("Saving {0} total artist media".format(dbNumAlbums))
        saveFile(idata=dbdata, ifile=savename)

        self.createArtistModValMetadata(modVal=modVal, db=dbdata, debug=debug)
        self.createArtistAlbumModValMetadata(modVal=modVal, db=dbdata, debug=debug)

    return saveIt


masterDBDF[~(masterDBDF.DiscArtist == masterDBDF.CleanDiscArtist)]
data = getFile("creditToDownload.p")
%load_ext autoreload
%autoreload
from masterArtistNameDB import masterArtistNameDB
manDB = masterArtistNameDB("master")
manDB.removeArtist("NF")
manDB.removeArtist("nf")
dbrs = getFile("dbrenames.yaml")
del dbrs["<NAME>"]
manDB.addRenames(dbrs)
manDB.addRenames(getFile("dbrenames.yaml"))
manDB.save()
manDB.saveRenames()
start,cmt=clock("Parsing Single ModVal")
pdb.parse(db="Discogs", modVals=[0], nProcs=1, force=False)
elapsed(start, cmt)
data = getFile('/Users/tgadfort/Music/Discog/artists-discogs-db/0-DB.p')
len(data)
def getArtistNumAlbums(artistData):
    numAlbums = sum([len(x) for x in artistData.media.media.values()])
    return numAlbums
saveIt = True
dbNumAlbums = sum([getArtistNumAlbums(artistData) for artistData in data.values()])
print('0', dbNumAlbums)

bsdata = getHTML("/Volumes/Piggy/Discog/artists-allmusic/99/0002579199.p")
#bsdata = getHTML("/Users/tgadfort/Downloads/DJ Spinz _ Credits _ AllMusic.htm")
data = getFile("/Users/tgadfort/Music/Discog/artists-allmusic-db/99-DB.p")
data['0002579199']
amcredit = artistAMCredit()
#amdata = amcredit.getData("/Users/tgadfort/Downloads/DJ Spinz _ Credits _ AllMusic.htm")
amdata = maindb.dbdata["AllMusic"]["Artist"].getData("/Volumes/Piggy/Discog/artists-allmusic/99/0002579199.p")
amdata.show()
dbdata["LastFM"]["Artists"].parseArtistModValExtraFiles(0, force=True, debug=True)
from glob import glob
glob('/Volumes/Piggy/Discog/artists-lastfm/0/extra/11792020000-*.p')
bsdata = getHTML(ifile)
ols = bsdata.findAll("ol", {"class": "buffer-standard"})
ols
ifile='/Volumes/Piggy/Discog/artists-lastfm/0/extra/72742897500-2.p'
#getFile(ifile)
a       = dbdata["LastFM"]["Artist"].getData(ifile, debug=False)

def info(a):
    name    = a.artist.name
    ID      = a.ID.ID
    url     = a.url.url
    pages   = a.pages
    media   = a.media
    counts  = a.mediaCounts
    profile = a.profile

    print("{0: <10}{1}".format("Name:", name))
    print("{0: <10}{1}".format("ID:", ID))
    print("{0: <10}{1}".format("URL:", url))
    print("{0: <10}{1}".format("Pages:", pages.get()))
    print("{0: <10}{1}".format("Profile:", profile.get()))
    print("{0: <10}{1}".format("Counts:", counts.get()))
    print("{0: <10}".format("Media:"))
    for mediaType,mediaData in media.get()['media'].items():
        for mediaInfo in mediaData:
            print("\t{0: <10}{1}".format(mediaType, mediaInfo.get()['album']))
info(a)
nProcs = 8
%load_ext autoreload
%autoreload
from mainDB import mainDB
maindb = mainDB(mdb=None, create=True, debug=True)
start,cmt=clock("Parsing All Artists")
start2,cmt2=clock("Parallel Artist Parsing")

parseArtistsParallel(db="AllMusic", nProcs=nProcs)
maindb.setDBFull("AllMusic")

parseArtistsParallel(db="MusicBrainz", nProcs=nProcs)
maindb.setDBFull("MusicBrainz")

parseArtistsParallel(db="Discogs", nProcs=nProcs)
maindb.setDBFull("Discogs")

#parseArtistsParallel(db="AceBootlegs", nProcs=1)
#parseArtistsParallel(db="DatPiff", nProcs=1)
#parseArtistsParallel(db="RateYourMusic", nProcs=3)
parseArtistsParallel(db="LastFM", nProcs=nProcs)
maindb.setDBFull("LastFM")

#parseArtistsParallel(db="RockCorner", nProcs=3)
#parseArtistsParallel(db="CDandLP", nProcs=3)
#parseArtistsParallel(db="MusicStack", nProcs=1)
elapsed(start2, cmt2)
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Master DB<jupyter_code>print("\n\n{0}\n".format("Full DB"))
print("\n\n{0}\n".format("Known DB"))
#maindb.setDBKnown() ## Do this to recreate everything
#maindb.setDBKnown()
#artistDBs = maindb.getKnownArtistDBs()<jupyter_output><empty_output><jupyter_text>Download My Known Artists<jupyter_code>mdb = myMusicDBMap(debug=True)
dbsToGet = ["AllMusic", "Discogs", "MusicBrainz", "LastFM", "RockCorner", "CDandLP"]
searches = [dbdata["AllMusic"]["Artists"].searchAllMusicForArtist, dbdata["Discogs"]["Artists"].searchDiscogForArtist,
            dbdata["MusicBrainz"]["Artists"].searchMusicBrainzForArtist, dbdata["LastFM"]["Artists"].searchLastFMForArtist,
            dbdata["RockCorner"]["Artists"].searchRockCornerForArtist, dbdata["CDandLP"]["Artists"].searchCDandLPForArtist]

dbsToGet = ["AllMusic"]
searches = [dbdata["AllMusic"]["Artists"].searchAllMusicForArtist]
def searchAll(mdb, minI=-1):
    nArtists = len(mdb.getArtists())
    for i,artistName in enumerate(mdb.getArtists()):
        if i <= minI:
            continue

        print('\n\n\n')
        print('-'*130)
        print(i,'/',nArtists,'===>\t',artistName)
        print('-'*130)
        print('\n')
        
        dbdata = mdb.getArtistData(artistName)
        if all([dbdata.get("LastFM"), dbdata.get("Discogs"), dbdata.get("AllMusic"), dbdata.get("MusicBrainz")]):
            print("{0} is well known.".format(artistName))
            continue
        else:
            print("{0} get it!".format(artistName))

        for search in searches:
            print("====>",search)
            try:
                search(artistName)
            except:
                sleep(3)
searchAll(mdb, 3404)
# Refused<jupyter_output><empty_output><jupyter_text>Albums<jupyter_code>%load_ext autoreload
%autoreload
from album import album
from albums import albums
from discogsBase import discogs

disc = discogs()
albs = albums(disc)
#albs.parseAlbumModValFiles(modVal=27)
def parseAlbums(modVal):
    albs.parseAlbumModValFiles(modVal=modVal)
    #albs.createAlbumModValMetadata(modVal)
    #albs.createAlbumModValMetadata(modVal=modVal)

def parseAlbumsParallel(modVals=None):
    if modVals is None:
        modVals = ["NAN"] + list(range(100))
        #modVals = range(100)
    nProcs = 2
    pool = Pool(processes=nProcs)
    result = pool.map_async(parseAlbums, modVals)
 
    start, cmt = clock("Processing {0} modVals using {1} processes".format(modVals, nProcs))
    print("Runing .",end="")
    while not result.ready():
        #print(".",end="")
        time.sleep(10)
    print("")
    
    elapsed(start, cmt)
    return result.get()
%load_ext autoreload
%autoreload
from album import album
from albums import albums
albs = albums(disc)

start,cmt=clock("Parsing All Albums")
start2,cmt2=clock("Parallel Album Parsing")
parseAlbumsParallel()
elapsed(start2, cmt2)
elapsed(start, cmt)<jupyter_output><empty_output><jupyter_text>Download Stuff<jupyter_code>from dbArtistsDeezer         import dbArtistsDeezer
from dbArtistsLastFM         import dbArtistsLastFM
from dbArtistsDiscogs        import dbArtistsDiscogs
from dbArtistsAllMusic       import dbArtistsAllMusic
from dbArtistsMusicBrainz    import dbArtistsMusicBrainz
from dbArtistsAlbumOfTheYear import dbArtistsAlbumOfTheYear
from time import sleep

class downloadDBData:    
    def __init__(self):
        self.dbs = {}
        self.dbs["Deezer"]         = dbArtistsDeezer()
        self.dbs["AlbumOfTheYear"] = dbArtistsAlbumOfTheYear()
        self.dbs["LastFM"]         = dbArtistsLastFM()
        self.dbs["Discogs"]        = dbArtistsDiscogs()
        self.dbs["MusicBrainz"]    = dbArtistsMusicBrainz()
        self.dbs["AllMusic"]       = dbArtistsAllMusic()
        
        self.maxArtists = {}
        self.maxArtists["Deezer"]         = 2
        self.maxArtists["AlbumOfTheYear"] = 2
        self.maxArtists["LastFM"]         = 2
        self.maxArtists["Discogs"]        = 20
        self.maxArtists["MusicBrainz"]    = 20
        self.maxArtists["AllMusic"]       = 20

        self.ignoreDBList = []
        
    def ignoreDB(self, db):
        if self.dbs.get(db) is not None:
            self.ignoreDBList.append(db)
        else:
            print("Could not add {0} to the ignore list".format(db))
        
        
    def downloadArtistData(self, artist, show=False):
        for db in self.dbs.keys():
            if db not in self.ignoreDBList:
                self.downloadArtistDBData(artist, db, show=show)
        
    def downloadArtistDBData(self, artist, db, show=False):
        if show is True:
            print("="*50,artist,"="*50)
        try:
            self.dbs[db].searchForArtist(artist,force=False,maxArtists=self.maxArtists[db],debug=False)
        except:
            print("Error downloading data for {0} from {1}".format(artist, db))
        sleep(1.5)
    
    def downloadArtistsData(self, toget, byDB=False):
        print(toget)
        if not isinstance(toget, list):
            toget=[toget]
        if byDB is True:
            for db in self.dbs.keys():
                if db in self.ignoreDBList:
                    continue
                for i,artist in enumerate(toget):
                    print("="*50,artist,' ({0}/{1})'.format(i,len(toget)),"="*50)
                    self.downloadArtistDBData(artist, db)
        else:
            for i,artist in enumerate(toget):
                for db in self.dbs.keys():
                    if db in self.ignoreDBList:
                        continue
                    print("="*50,artist,' ({0}/{1})'.format(i,len(toget)),"="*50)
                    self.downloadArtistDBData(artist, db)
toget=['山岡晃 [Akira Yamaoka]', '<NAME> Garfunkel', 'Palace Brothers', 'Bowie', 'Kan Gao', 'Palace Music', 'Ultrasound', 'The Windbreakers', 'Invisible', 'Joe Byrd and The Field Hippies', 'Misophone', 'Tuff Monks', 'The Castaways', 'Irene', 'The Untamed Youth', 'The Mockingbirds', 'The Novas', 'The Blue Birds', 'The Chymes', 'The Bittersweets', 'The Wedge', 'Fryars', 'The Pharaohs [psycho]', 'Ofo the Black Company', '<NAME> and The Dap-Kings', '<NAME>', 'Stefano Testa', "Christine's Cat", 'Eighth Penny Matter', 'Frank Sumatra and the Mob', 'Clothilde', 'Animals That Swim', 'Le Stelle di <NAME>', 'ดอน สอนระเบียบ [Don Sonrabiap]', 'Minta', 'Captain Groovy and His Bubblegum Army', 'The Children of Sunshine', 'Bagarre', '久石譲 [Joe Hisaishi]', '<NAME>', 'Methusalem', 'Semáforo', 'Mr Blobby', 'Ahab and The Wailers', '22-Pistepirkko', "<NAME>", 'Peach and Lee', 'The Nightblooms', 'Альянс [Aljans]', 'Aventuras de Kirlian', 'Moniker', 'The Bats', 'Drugstore', 'Juana La Loca', 'u.n.p.o.c.', 'Spınal Tap', 'Os Ekos', 'Superette', 'Jass Carnival', 'Monks', 'The Pentangle', 'Tomorrow', 'Heróis do Rock', 'Dogs Die in Hot Cars', 'The Squires', 'Dover', 'Max Frost and The Troopers', 'The Times', '<NAME>', 'Tranzmitors', 'The Birdmen of Alkatraz', '<NAME> and The Egyptians', 'Alice', '20/20', 'Reigndance', 'The Gandharvas', '<NAME>', 'The Saturday Nights', 'Viviane', '<NAME>', 'Alexander "Skip" Spence', 'The Eyes of Mind', 'Mclusky', 'Clinic', "The Would Be's", 'Jáfumega', 'Los 007', 'Hefner', 'LiLiPUT', 'Théâtre du chêne noir', 'La Cofradía de la Flor Solar', 'Bartlebees', 'Tayti', 'The Hummingbirds', 'Weekend', 'The Brotherhood of Lizards', 'The Partisan Seed', 'Gomo', 'The Spades', '<NAME>', 'LMNOP', 'Cardinal', 'Marante', 'Jeanette', 'Triumph the Insult Comic Dog', 'The Sharades', '<NAME>', 'Um Zero Amarelo', 'WAR', "Pop Dell'arte", 'Quinta do Bill', 'The Belairs', 'The Passions', 'Idlewild', 'Chick Lewis', 'Wah! Heat', '<NAME>', 'Wolfhounds', 'The Inmates', "Trixie's Big Red Motorbike", 'The Californians', 'The Sons of Adam', 'The Paragons', 'Finders Keepers', 'Salada de Frutas', 'Major Organ and the Adding Machine', '<NAME> and The Green Fuz', 'Sofia de Portugal', "O'queStrada", 'Zlad!', 'Melancholic Youth of Jesus', 'The Pharos', 'Reparata and The Delrons', 'Y Diliau', '<NAME>', '<NAME>', 'The Fender IV', 'The Crossfires', 'Eleri Llwyd', 'The Jefferson Handkerchief', 'Dinosaur', 'The Phantom', '<NAME>', '<NAME>', 'រស់សេរីសុទ្ធា [Ros Sereysothea]', 'McCarthy', 'The King of Luxembourg', 'The Union Boys', 'Francois Peglau', 'Acid Casuals', 'Les Yper Sound', '<NAME>', 'The Boy Toys', 'The Whyte Boots', '喜納昌吉 [Shoukichi Kina]', 'Polaris', 'The Montgolfier Brothers', 'The Fans', 'Acid Gallery', 'Arrah and the Ferns', 'Butcher Boy', 'Beasts of Bourbon', 'Betty and the Werewolves', 'The Fish', 'Biblical Names', 'the Paranoid Android', 'The Ways', 'Chailai Chaiyatha]', 'ดวงดาว มนต์ดารา', 'ไฉไล ไชยทา [Duangdao Mondara', 'Famous [UK]', 'Secos', 'Molhados', 'เบิร์ด-เสก [Bird', 'Sek]', 'cybelle', 'lyme', 'The Voidoids', 'BBQ Show', 'The King Khan', 'His Shrines', 'the Bad', 'the Queen', 'The Good', 'Mitch', 'Richard', 'The Cosmonauts', '4.40']
%autoreload
#from downloadDBData import downloadDBData
ddbd = downloadDBData()
ddbd.ignoreDB("Deezer")
ddbd.downloadArtistsData(toget, byDB=True)
mdb = myMusicDBMap(debug=True)
maindb = mainDB(mdb=mdb, create=False, debug=True)
dbdata = maindb.dbdata
ifile = "unmatchedArtists4.p"
noMatches = getFile(ifile)
print(len(noMatches))
dbs = ['Discogs', 'AllMusic', 'MusicBrainz', 'LastFM']
dbs = ['MusicBrainz']

#noMatches=['Go-A', 'PM', 'Qango']

from time import sleep
maxI = -1
for i,artist in enumerate(noMatches):
    print("="*100)
    print("="*100)
    print("="*100)
    print("="*100)
    print(i,'/',len(noMatches),'\t',artist)
    if i <= maxI:
        continue
    for db in dbs:
        print("="*100)
        print(i,'/',len(noMatches),'\t',artist,'===>',db)
        try:
            dbdata[db]["Artists"].searchForArtist(artist)
        except:
            sleep(5)
from searchUtils import findPatternExt
from fileUtils import getBaseFilename
vals=[getBaseFilename(x) for x in findPatternExt("/Volumes/Piggy/Charts/data/spotify/categories", "-se-", ".csv")]
[x.split('-')[3:] for x in vals][:3]

#url="https://spotifycharts.com/regional/se/weekly/2021-01-29--2021-02-05/download"
from pandas import date_range
#2016-12-23--2016-12-30
#2017-01-06--2017-01-13
from datetime import datetime
start = '12/23/2016'
end   = datetime.now().strftime("%Y-%m-%d")
datevals = [x.strftime('%Y-%m-%d') for x in list(date_range(start=start, end=end, freq='W-FRI'))]
files = findPatternExt("/Users/tgadfort/Downloads/spotify", "regional", ".csv")
print([x.split("-")[1] for x in files])
bsdata = getHTML("../../charts/playback/one-hit-wonders-1970s.p")
bsdata
div = bsdata.find("div", {"class": "decade-list"})
if div is not None:
    songdivs = div.findAll("div")
    for songdiv in songdivs:
        span = songdiv.find("span")
        artist = None
        if span is not None:
            artist = span.text
        refs = songdiv.findAll("a")
        print(refs)
        print(artist)
        break<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1464499536217219, 'fraction_numerical': 0.01366051100430053, 'mean_word_length': 4.4720267620256084, 'pattern_counts': {'":': 2, '<': 153, '<?xml version=': 0, '>': 162, 'https://': 106, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '29', 'dir': 'jupyter-structured-clean-dedup', 'id': '78011', 'license': 'MIT', 'n_tokens_mistral': 17514, 'n_tokens_neox': 16862, 'n_words': 3138, 'path': 'notebooks/Metadata.ipynb', 'repo_name': 'tgadf/dbmaster', 'size': '47554'}