<jupyter_start><jupyter_text>Load Music DBs<jupyter_code>#mcm    = matchChartMusic(mdb)
recreate = True
if recreate:
    maindb = mainDB(mdb=None, create=True, debug=True)
    maindb.setDBArtists(recreate=False)
    maindb = mainDB(mdb=None, create=False, debug=False)
    maindb.loadDBDataMap()
else:
    maindb = mainDB(mdb=None, create=False, debug=False)
    maindb.loadDBDataMap()    
_, _ = clock("Last Run")<jupyter_output>Setting Basic Database Objects
=========================   discogs   =========================
=========================   allmusic   =========================
=========================   musicbrainz   =========================
=========================   lastfm   =========================
=========================   rockcorner   =========================
=========================   acebootlegs   =========================
=========================   cdandlp   =========================
=========================   rateyourmusic   =========================
=========================   musicstack   =========================
Database Records:
  Creating Database Records for Discogs
  Creating Database Records for AllMusic
  Creating Database Records for MusicBrainz
  Creating Database Records for LastFM
  Creating Database Records for RockCorner
  Creating Database Records for AceBootlegs
  Creating Database Records for CDandLP
  Creating Database Records for RateYourMusic
  Creating Databas[...]<jupyter_text>Load My Matched Music DB<jupyter_code>manDB   = masterArtistNameDB("main", init=False)
saveFile(idata=manDB.getRenames(), ifile="main.yaml")
mdbmap  = None
mdbmaps = {}
mdbmaps["Music"] = musicDBMap("Music", init=False, copy=False)
def updateManDB(step=1):
    if step == 1:
        tmpmanDB = masterArtistNameDB("main", init=True)
        tmpmanDB.forceReload(getFile("main.yaml"))
        #manDB.addRenames(getFile("relDBRenames2.yaml"))
        tmpmanDB.checkForRecursives()

    if step == 2:
        tmpmanDB = masterArtistNameDB("main", init=True)
        tmpmanDB.forceReload(getFile("main.yaml"))
        #manDB.addRenames(getFile("relDBRenames2.yaml"))
        tmpmanDB.save()
        
        manDB = masterArtistNameDB("main", init=False)
        saveFile(idata=manDB.getRenames(), ifile="main.yaml")
        
    if step == 3:
        tmpmanDB = masterArtistNameDB("main", init=False)
        tmpmanDB.addRenames(getFile("relDBRenames2.yaml"))
        tmpmanDB.checkForRecursives()
        
    if step == 4:
        tmpmanDB = masterArtistNameDB("main", init=False)
        tmpmanDB.addRenames(getFile("relDBRenames2.yaml"))
        tmpmanDB.save()
        
        manDB = masterArtistNameDB("main", init=False)
        saveFile(idata=manDB.getRenames(), ifile="main.yaml")
        
    
    _, _ = clock("Last Ran")
updateManDB(2)<jupyter_output>========================= masterArtistNameDB =========================
  Loading data from /Users/tgadfort/opt/anaconda3/envs/py37/musicnames/mainArtistNameDB.p
  Initializing a fresh DB for main
  There are currently 0 artist keys.
  There are currently 0 renamed artist keys.
  Forced reload of 6752 artist names
Saving 6752 artist keys to /Users/tgadfort/opt/anaconda3/envs/py37/musicnames/mainArtistNameDB.p
Saving data to /Users/tgadfort/opt/anaconda3/envs/py37/musicnames/mainArtistNameDB.p
  --> This file is 177.4kB.
Saved data to /Users/tgadfort/opt/anaconda3/envs/py37/musicnames/mainArtistNameDB.p
  --> This file is 177.4kB.
========================= masterArtistNameDB =========================
  Loading data from /Users/tgadfort/opt/anaconda3/envs/py37/musicnames/mainArtistNameDB.p
  There are currently 6752 artist keys.
  There are currently 7820 renamed artist keys.
Current Time is Wed Jan 20, 2021 19:32:16 for Last Ran<jupyter_text>Find My Music<jupyter_code>from findMyMusic import findMusic, getMusicStatus
mmb = findMusic()
mmb, mmm, unknownArtists = getMusicStatus(mdbmap=mdbmaps["Music"], mmb=mmb)
if False:
    dbDumpData = getFile("dbDump.p")
    len(dbDumpData)

    for artistName in mmb.getArtists():
        primaryKey = mdbmaps["Music"].getPrimaryKey(artistName=artistName, artistID=None)
        artistDumpData = dbDumpData[primaryKey]
        mdbmaps["Music"].addArtistByKey(primaryKey, artistName, None)
        for db,dbID in artistDumpData.items():
            mdbmaps["Music"].addArtistDataByKey(primaryKey, db, dbID)    
            
    mdbmaps["Music"].save()
def addDiscogs(mdbmaps, chartType, amID, dbID):
    if dbID is None:
        return
    mdbmaps[chartType].addArtistDataByKey(amID, "Discogs", dbID)

def addAllMusic(mdbmaps, chartType, amID, dbID):
    if dbID is None:
        return
    dbID = dbID[2:]
    mdbmaps[chartType].addArtistDataByKey(amID, "AllMusic", dbID)
    print("Set {0} <-> {1} for AllMusic".format(dbID,amID))

def addMusicBrainz(mdbmaps, chartType, amID, url):
    try:
        int(url)
        dbID = url
    except:
        dbID = maindb.getArtistDBIDFromUtil("MusicBrainz", url)
    if dbID is None:
        return
    mdbmaps[chartType].addArtistDataByKey(amID, "MusicBrainz", dbID)
    
    
def manualAppends(unknownArtists, chartType="Music", minAlbums=0, add=True):
    for artistName in unknownArtists.keys():
        if add:
            idx = mdbmaps[chartType].getPrimaryKey(artistName=artistName)
            print("mdbmaps[{0}].addArtistByName({1})".format("\"{0}\"".format(chartType), "\"{0}\"".format(artistName)))
            print("add{0}(mdbmaps, {1}, {2}, {3})   ### {4}".format("AllMusic", "\"{0}\"".format(chartType), "\"{0}\"".format(idx), "\"{0}\"".format(""), artistName))
            print("#add{0}(mdbmaps, {1}, {2}, {3})    ### {4}".format("Discogs", "\"{0}\"".format(chartType), "\"{0}\"".format(idx), "\"{0}\"".format(""), artistName))
            print("")
        else:
            print("ignores.append(\"{0}\")".format(artistName))
            if " & " in artistName:
                continue
                print("ignores.append(\"{0}\")".format(artistName))

    print("mdbmaps[{0}].save()".format("\"{0}\"".format(chartType)))
    
    
manualAppends(unknownArtists)
mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "ff2ef07dd55857f762abfc141cc0dd4f", "mn0000151290")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "ff2ef07dd55857f762abfc141cc0dd4f", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("C-Murder")
addAllMusic(mdbmaps, "Music", "f2c67e3b30223cda64dd95bb209b14b8", "mn0000527773")   ### C-Murder
#addDiscogs(mdbmaps, "Music", "f2c67e3b30223cda64dd95bb209b14b8", "")    ### C-Murder

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "073ca4761ccce4166ae76abdab79262c", "mn0000183512")   ### Lil' Flip
#addDiscogs(mdbmaps, "Music", "073ca4761ccce4166ae76abdab79262c", "")    ### <NAME>

mdbmaps["Music"].save()
mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "83c2fadabaa723af0b73db4fc950d1e6", "mn0003747024")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "83c2fadabaa723af0b73db4fc950d1e6", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "45a5064d057441062ea4e21d7042ca7a", "mn0000885569")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "45a5064d057441062ea4e21d7042ca7a", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("Sleater-Kinney")
addAllMusic(mdbmaps, "Music", "cef44853aaf30a724982dc8482074606", "mn0000026164")   ### Sleater-Kinney
#addDiscogs(mdbmaps, "Music", "cef44853aaf30a724982dc8482074606", "")    ### Sleater-Kinney

mdbmaps["Music"].save()
def analyzeMatches(maindb, mdbmaps, chartType="Music", maxVal=1.0, diffVal=0.1):
    dbRenames = {}
    toget = {}
    for primaryKey,artistName in mdbmaps[chartType].getArtists().items():
        primaryArtistName = artistName
        artistData = mdbmaps[chartType].getArtistDataByKey(primaryKey)
        for db,dbID in artistData.getDict().items():
            if db in ["DatPiff", "MetalStorm", "LastFM", "CDandLP", "AceBootlegs", "RockCorner", "MusicStack", "RateYourMusic"]:
                continue
            if dbID is not None:
                secondaryArtistName = maindb.getArtistDBNameFromID(db, dbID)
                if secondaryArtistName is None:
                    if toget.get(db) is None:
                        toget[db] = []
                    toget[db].append(primaryArtistName)
                    continue
                s = SequenceMatcher(None, primaryArtistName, secondaryArtistName)
                ratio = s.ratio()
                maxRatio = maxVal
                if ratio >= maxRatio:
                    continue

                if ratio >= maxRatio-diffVal:
                    if dbRenames.get(secondaryArtistName) is not None:
                        if dbRenames[secondaryArtistName] == primaryArtistName:
                            continue
                        else:
                            pass
                    dbRenames[secondaryArtistName] = primaryArtistName
                    print("[{0: <30} {1: <4} {2: >30}] \t --> ({3}) {4} / {5} ({6})".format(primaryArtistName,round(ratio,2),secondaryArtistName,chartType,primaryKey,dbID,db))


    print("Found {0} artists to get.".format(len(toget)))
    print("Found {0} artists to rename.".format(len(dbRenames)))
    return toget, dbRenames



def analyzeRenames(manDB, dbRenames):
    redos = {}
    dels = []
    if len(dbRenames) == 0:
        print("Nothing to process.")
        return
    for oldername,bestname in dbRenames.items():
        if oldername in manDB.artistNameDB.keys() and bestname not in manDB.artistNameDB.keys():
            print(oldername,bestname)

            redos[bestname] = oldername
            dels.append(oldername)
            print('\t',oldername,'\t',bestname)

    print("# Renames: {0}".format(len(dbRenames)))
    print("# Dels: {0}".format(len(dels)))
    for oldername in dels:
        del dbRenames[oldername]

    print("# Redos: {0}".format(len(redos)))
    dbRenames.update(redos)
    print("# Renames: {0}".format(len(dbRenames)))
    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")




    def isAscii(ele):
        return len(ele) == len(ele.encode())

    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if isAscii(v) and not isAscii(k):
            if '’' in k or "“" in k:
                continue
            redos[v] = k
            dels.append(k)
            print('\t',k,'\t',v)
    #print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    #print(len(dbRenames))
    dbRenames.update(redos)
    #print(len(dbRenames))
    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")




    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if " the " in v and " The " in k:
            if k == v.replace(" the ", " The "):
                redos[v] = k
                dels.append(k)
                print('\t',k,'\t',v)

    #print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    #print(len(dbRenames))
    dbRenames.update(redos)
    #print(len(dbRenames))
    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")




    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if '’' in v or "“" in v:
            if not '’' in k and not "“" in v:
                redos[v] = k
                dels.append(k)
                print('\t',k,'\t',v)

    #print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    #print(len(dbRenames))
    dbRenames.update(redos)
    #print(len(dbRenames))


    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")








    def isAscii(ele):
        return len(ele) == len(ele.encode())

    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if isAscii(v) and not isAscii(k):
            if '’' in k or "“" in k:
                continue
            redos[v] = k
            dels.append(k)
            print('\t',k,'\t',v)
    #print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    #print(len(dbRenames))
    dbRenames.update(redos)
    #print(len(dbRenames))


    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")








    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if " the " in v and " The " in k:
            if k == v.replace(" the ", " The "):
                redos[v] = k
                dels.append(k)
                print('\t',k,'\t',v)

    #print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    #print(len(dbRenames))
    dbRenames.update(redos)
    #print(len(dbRenames))


    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")







    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if '’' in v or "“" in v:
            if not '’' in k and not "“" in v:
                redos[v] = k
                dels.append(k)
                print('\t',k,'\t',v)

    #print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    #print(len(dbRenames))
    dbRenames.update(redos)
    #print(len(dbRenames))


    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")
    #print(len(dbRenames))
    
    
    

def removeDBRenames(mdbmaps, chartType, dbRenames):
    for k,v in dbRenames.items():
        kKey = mdbmaps[chartType].getPrimaryKey(artistName=k, artistID=None)
        if kKey is not None:
            mdbmaps[chartType].removeArtistByKey(kKey)

        kKey = mdbmaps[chartType].getPrimaryKey(artistName=v, artistID=None)
        if kKey is not None:
            mdbmaps[chartType].removeArtistByKey(kKey)
    mdbmaps[chartType].save()

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "945c0dd5f79106c6c4dc7b2a9ce3ee83", "mn0000819535")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "945c0dd5f79106c6c4dc7b2a9ce3ee83", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "f1cb57c0647edaed5826c3dcdcc5ba5a", "mn0000074397")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "f1cb57c0647edaed5826c3dcdcc5ba5a", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "b738e544cd377297e91d86f691b2ea46", "mn0000211483")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "b738e544cd377297e91d86f691b2ea46", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "f99308cca5a3e1a3530eeb8ce487e572", "mn0002937107")   ### Chief Keef
#addDiscogs(mdbmaps, "Music", "f99308cca5a3e1a3530eeb8ce487e572", "")    ### Chief Keef

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "effdb2e872494faf9bb724bf945089e6", "mn0000631790")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "effdb2e872494faf9bb724bf945089e6", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("El Gran Combo De Puerto Rico")
addAllMusic(mdbmaps, "Music", "ef920fec24d05de4b830a854fd4b8daf", "mn0000789514")   ### El Gran Combo De Puerto Rico
#addDiscogs(mdbmaps, "Music", "ef920fec24d05de4b830a854fd4b8daf", "")    ### El Gran Combo De Puerto Rico

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "69db9f9ce6e81a9c1bf3fbfe07bb9d09", "mn0000050034")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "69db9f9ce6e81a9c1bf3fbfe07bb9d09", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "2fd2aeaaf80a0b618e69c478ced29cfd", "mn0000295276")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "2fd2aeaaf80a0b618e69c478ced29cfd", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "29d5393c9e3c86aedde59a0e2031e99e", "mn0000812046")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "29d5393c9e3c86aedde59a0e2031e99e", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "b4b239893617275be8e0dbdfa00cb75f", "mn0000294923")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "b4b239893617275be8e0dbdfa00cb75f", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "1c8f2d8cd8df8441923957785b002c21", "mn0000066570")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "1c8f2d8cd8df8441923957785b002c21", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("Kitsuné")
addAllMusic(mdbmaps, "Music", "817d7182cdc653644dd1fc579f1e92db", "mn0000771695")   ### Kitsuné
#addDiscogs(mdbmaps, "Music", "817d7182cdc653644dd1fc579f1e92db", "")    ### Kitsuné

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "<KEY>", "mn0000489520")   ### <NAME>
#addDiscogs(mdbmaps, "Music", "<KEY>", "")    ### <NAME>

mdbmaps["Music"].addArtistByName("<NAME> & Crazy Horse")
addAllMusic(mdbmaps, "Music", "b420462692ee5970234689b2a1ddea84", "mn0000860578")   ### Neil Young & Crazy Horse
#addDiscogs(mdbmaps, "Music", "b420462692ee5970234689b2a1ddea84", "")    ### Neil Young & Crazy Horse

mdbmaps["Music"].addArtistByName("Nerve")
#addAllMusic(mdbmaps, "Music", "7f50302cbfc598f5466704cbe68d4b84", "")   ### Nerve
addDiscogs(mdbmaps, "Music", "7f50302cbfc598f5466704cbe68d4b84", "93005")    ### Nerve

mdbmaps["Music"].addArtistByName("Nitty Gritty Dirt Band")
addAllMusic(mdbmaps, "Music", "edc43e9780289c1dde63f080f4767862", "mn0000718907")   ### Nitty Gritty Dirt Band
#addDiscogs(mdbmaps, "Music", "edc43e9780289c1dde63f080f4767862", "")    ### Nitty Gritty Dirt Band

mdbmaps["Music"].addArtistByName("O.A.R")
addAllMusic(mdbmaps, "Music", "9c77c51624b108eef43a0e979cd14ef5", "mn0000884449")   ### O.A.R
#addDiscogs(mdbmaps, "Music", "9c77c51624b108eef43a0e979cd14ef5", "")    ### O.A.R

mdbmaps["Music"].addArtistByName("TV on the Radio")
addAllMusic(mdbmaps, "Music", "c141d80aac65375fb6fa84bacb182498", "mn0000012972")   ### TV on the Radio
#addDiscogs(mdbmaps, "Music", "c141d80aac65375fb6fa84bacb182498", "")    ### TV on the Radio

mdbmaps["Music"].addArtistByName("<NAME>")
addAllMusic(mdbmaps, "Music", "3e80128381a093793bee8eff272c5a9d", "mn0000790604")   ### Taj Mahal
#addDiscogs(mdbmaps, "Music", "3e80128381a093793bee8eff272c5a9d", "")    ### Taj Mahal

mdbmaps["Music"].addArtistByName("Widespread Panic")
addAllMusic(mdbmaps, "Music", "58ba3c0cfdd58d61c1c085856f56e2f0", "mn0000257141")   ### Widespread Panic
#addDiscogs(mdbmaps, "Music", "58ba3c0cfdd58d61c1c085856f56e2f0", "")    ### Widespread Panic

mdbmaps["Music"].addArtistByName("Yoko Ono")
addAllMusic(mdbmaps, "Music", "2ce38547dfbced809b6343246fdb9a09", "mn0000521704")   ### Yoko Ono
#addDiscogs(mdbmaps, "Music", "2ce38547dfbced809b6343246fdb9a09", "")    ### Yoko Ono

mdbmaps["Music"].addArtistByName("Yung Lean")
addAllMusic(mdbmaps, "Music", "a2d70c1b64d7c9c513ae530eb40abf4c", "mn0003322070")   ### Yung Lean
#addDiscogs(mdbmaps, "Music", "a2d70c1b64d7c9c513ae530eb40abf4c", "")    ### Yung Lean

mdbmaps["Music"].save()
toget, dbRenames = analyzeMatches(maindb, mdbmaps, "Music", maxVal=1.0, diffVal=1.0)
toget
removeDBRenames(mdbmaps, "Music", dbRenames)
analyzeRenames(manDB, dbRenames)
updateManDB(2)
updateManDB(4)
artistAlbumData = {}
for artistName in mmb.getArtists():
    albumsDBData  = mmb.getArtistAlbumsByArtist(artistName)
    knownAlbumDBs = ["Match", "Random"]
    artistAlbums  = []
    for knownAlbumDB in knownAlbumDBs:
        for albumDir,albums in albumsDBData[knownAlbumDB].getAlbums().items():
            artistAlbums += albums
    artistAlbumData[artistName] = list(set(artistAlbums))
print("Found {0} artist albums data".format(len(artistAlbumData)))
mdbmaps = {}
mdbmaps[chartType] = musicDBMap(chartType, init=True, copy=False)
mdbmc = masterDBMatchClass(maindb, mdbmaps)
def reMatch(mdbmaps=mdbmaps, mdbmc=mdbmc):
    matchData = {mdbmaps[chartType].getPrimaryKey(artistName=artistName, artistID=None): {"ArtistName": artistName, "ArtistAlbums": artistAlbums} 
                 for artistName,artistAlbums in artistAlbumData.items()}
    mdbmc.setDBMatchData(chartType, matchData)
reMatch()
#mbDF = mdbmc.getMasterDF(chartType)
from functools import partial

def getThresholds(minAlbums=None):
    thresholds = {}
    thresholds[1000] = {'numArtistName': 1, 'artistNameCutoff': 0.95, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': int(200/8), 'score': 10.0}
    thresholds[500]  = {'numArtistName': 1, 'artistNameCutoff': 0.95, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': int(200/8), 'score': 5.0}
    thresholds[200]  = {'numArtistName': 1, 'artistNameCutoff': 0.95, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': int(200/8), 'score': 2.5}
    thresholds[100]  = {'numArtistName': 1, 'artistNameCutoff': 0.95, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': int(100/8), 'score': 1.5}
    thresholds[50]   = {'numArtistName': 2, 'artistNameCutoff': 0.95, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': int(50/8), 'score': 1.5}
    thresholds[20]   = {'numArtistName': 2, 'artistNameCutoff': 0.95, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': 3, 'score': 1.5}
    thresholds[10]   = {'numArtistName': 5, 'artistNameCutoff': 0.90, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': 2, 'score': 1.5}
    thresholds[5]    = {'numArtistName': 5, 'artistNameCutoff': 0.90, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': 2, 'score': 1.5}
    thresholds[3]    = {'numArtistName': 5, 'artistNameCutoff': 0.90, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': 2, 'score': 1.5}
    thresholds[2]    = {'numArtistName': 5, 'artistNameCutoff': 0.90, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': 1, 'score': 2.0}
    thresholds[1]    = {'numArtistName': 5, 'artistNameCutoff': 0.90, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': 1, 'score': 0.9}

    if minAlbums is None:
        return thresholds
    return thresholds[minAlbums]


def getIterItems():
    iterItems = {50: {"Max": 10000, "Vals": 50}}
    iterItems.update({20: {"Max": 50, "Vals": 100}})
    iterItems.update({10: {"Max": 20, "Vals": 200}})
    iterItems.update({5: {"Max": 10, "Vals": 250}})
    iterItems.update({3: {"Max": 5, "Vals": 500}})
    iterItems.update({2: {"Max": 3, "Vals": 500}})
    iterItems.update({1: {"Max": 2, "Vals": 500}})
    return iterItems


def getThresholdsWithoutAlbums(cutoff=0.9):
    return {'numArtistName': 5, 'artistNameCutoff': cutoff, 'artistAlbumCutoff': None, 'numArtistAlbums': None, 'score': None}
    


def matchItAll(mdbmaps, thresholds, useAlbums=True, mdbmc=None, db=None, toMatch=None):
    if toMatch is None:
        toMatch   = mdbmc.getDataToMatch(db, maxValues=10000, maxAlbums=50000)

    if len(toMatch) == 0:
        return

    num_processes = 3
    if useAlbums is True:
        func = matchDBArtistWithAlbums
    else:
        func = matchDBArtistWithoutAlbums
    pfunc = partial(func, **thresholds) # Giving some arguments for kwargs
    #argument_list = list(inputs.items()) # [random.randint(0, 100) for _ in range(num_jobs)]
    dbName = list(toMatch.keys())[0]
    copyMapData(mdbmaps[dbName])

    argument_list = toMatch[dbName]
    if len(argument_list) == 0:
        return
    print("Running imap multiprocessing for {0} artists ...".format(len(argument_list)))
    result_list = multiProc(func=pfunc, argument_list=argument_list,
                                           num_processes=num_processes)


    start, cmt = clock("Saving...")
    saveMapData(mdbmaps[dbName], result_list)
    elapsed(start, cmt)
    print("\nSleeping for 10 seconds...\n")
    #sleep(10)
    
    #mdbmc.matchMutualMaps()

    
def matchDBArtistWithoutAlbums(item, *args, **kwargs):
    primaryKey = item[0]
    artistData = item[1]
    artistName   = artistData["ArtistName"]
    artistID     = primaryKey
    artistAlbums = None
    mdbMatcher = matchDBArtist(maindb)
    mdbMatcher.setArtistInfo(artistName, artistID, artistAlbums)
    mdbMatcher.setThresholds(matchNumArtistName=kwargs['numArtistName'], matchArtistNameCutoff=kwargs['artistNameCutoff'],
                             matchArtistAlbumCutoff=kwargs['artistAlbumCutoff'], matchNumArtistAlbums=kwargs['numArtistAlbums'],
                             matchScore=kwargs['score']),
    mcs    = mdbMatcher.findPotentialArtistNameMatchesWithoutAlbums()
    retval = [primaryKey,artistName,artistID,mcs]
    return retval


def matchDBArtistWithAlbums(item, *args, **kwargs):    
    #time.sleep(0.0025)

    
    primaryKey = item[0]
    artistData = item[1]
    artistName   = artistData["ArtistName"]
    artistID     = primaryKey
    artistAlbums = artistData["ArtistAlbums"]
    
    mdbMatcher = matchDBArtist(maindb)
    mdbMatcher.setArtistInfo(artistName, artistID, artistAlbums)
    mdbMatcher.setThresholds(matchNumArtistName=kwargs['numArtistName'], matchArtistNameCutoff=kwargs['artistNameCutoff'], 
                             matchArtistAlbumCutoff=kwargs['artistAlbumCutoff'], matchNumArtistAlbums=kwargs['numArtistAlbums'],
                             matchScore=kwargs['score'])
    mcs    = mdbMatcher.findPotentialArtistAlbumMatches()
    retval = [primaryKey,artistName,artistID,mcs]
    return retval


def multiProc(func, argument_list, num_processes):
    pool = Pool(processes=num_processes)
    result_list_tqdm = []
    for result in tqdm(pool.imap(func=func, iterable=argument_list), total=len(argument_list)):
        result_list_tqdm.append(result)
    return result_list_tqdm


def copyMapData(mdbmap):
    mdbmap.saveCopy()

def saveMapData(mdbmap, result_list):
    for item in result_list:
        primaryKey = item[0] 
        artistName = item[1]
        artistID   = item[2]
        mcs        = item[3]
        mdbmap.addArtistByKey(primaryKey, artistName=artistName, artistID=artistID)
        for db,mc in mcs.items():
            matchID    = mc.matchID
            matchScore = mc.matchScore
            if matchID is not None:
                mdbmap.addArtistDataByKey(primaryKey, db, matchID)

    mdbmap.save()
    
    
    

def manualAppends(artistAlbumData, toMatch, chartType, minAlbums=0, add=True):
    for item in toMatch[chartType]:
        if add:
            albums = artistAlbumData[item[1]["ArtistName"]]
            if len(albums) < minAlbums:
                continue
            print("add{0}(mdbmaps, {1}, {2}, {3})   ### {4}".format("AllMusic", "\"{0}\"".format(chartType), "\"{0}\"".format(item[0]), "\"{0}\"".format(""), item[1]["ArtistName"]))
            print("add{0}(mdbmaps, {1}, {2}, {3})    ### {4}".format("Discogs", "\"{0}\"".format(chartType), "\"{0}\"".format(item[0]), "\"{0}\"".format(""), item[1]["ArtistName"]))
            print("### {0}\n".format("  ,  ".join(albums)))
        else:
            print("ignores.append(\"{0}\")".format(item[1]["ArtistName"]))
            if " & " in item[1]["ArtistName"]:
                continue
                print("ignores.append(\"{0}\")".format(item[1]["ArtistName"]))

    print("mdbmaps[chartType].save()")
    
    
for minAlbums,minAlbumsData in getIterItems().items():
    print("="*10,minAlbums)
    maxAlbums = minAlbumsData["Max"]
    maxValues = minAlbumsData["Vals"]
    for i in range(10):
        toMatch   = mdbmc.getDataToMatch(chartType, maxValues=maxValues, maxAlbums=maxAlbums, minAlbums=minAlbums, ignores=[])
        if len(toMatch[chartType]) == 0:
            break
        matchItAll(mdbmaps, getThresholds(minAlbums), toMatch=toMatch)
        
reMatch()
print("Done.\n\n")
toMatch   = mdbmc.getDataToMatch(chartType, maxValues=10000, maxAlbums=10, minAlbums=0, ignores=[])
len(toMatch[chartType])
matchItAll(mdbmaps, getThresholdsWithoutAlbums(1.0), toMatch=toMatch, useAlbums=False)
reMatch()
toMatch   = mdbmc.getDataToMatch(chartType, maxValues=10000, maxAlbums=100000, minAlbums=0, useKnown=False, dbMatches=0, ignores=[])
len(toMatch[chartType])
matchItAll(mdbmaps, getThresholdsWithoutAlbums(1.0), toMatch=toMatch, useAlbums=False)
reMatch()
extraAddsForMusic(mdbmaps, chartType)
toMatch   = mdbmc.getDataToMatch(chartType, maxValues=10000, maxAlbums=100000, minAlbums=0, useKnown=False, dbMatches=0, ignores=[])
len(toMatch[chartType])
toMatch   = mdbmc.getDataToMatch(chartType, maxValues=10000, maxAlbums=100000, minAlbums=0, useKnown=False, dbMatches=0, ignores=[])
len(toMatch[chartType])
matchItAll(mdbmaps, getThresholdsWithoutAlbums(1.0), toMatch=toMatch, useAlbums=False)
reMatch()
mdbmaps["Music"].getDF()
#mdbmc.getDF("Music")
manualAppends(artistAlbumData, toMatch, chartType, minAlbums=0, add=True)
mdbmaps[chartType].removeArtistByID("04f12d4923ec5069ac75e6d364b719cb")
mdbmaps[chartType].save()
toget, dbRenames = analyzeMatches(maindb, mdbmaps, chartType, maxVal=1.0, diffVal=1.0)
def addDiscogs(mdbmaps, chartType, amID, dbID):
    if dbID is None:
        return
    mdbmaps[chartType].addArtistDataByKey(amID, "Discogs", dbID)

def addAllMusic(mdbmaps, chartType, amID, dbID):
    if dbID is None:
        return
    dbID = dbID[2:]
    mdbmaps[chartType].addArtistDataByKey(amID, "AllMusic", dbID)

def addMusicBrainz(mdbmaps, chartType, amID, url):
    try:
        int(url)
        dbID = url
    except:
        dbID = maindb.getArtistDBIDFromUtil("MusicBrainz", url)
    if dbID is None:
        return
    mdbmaps[chartType].addArtistDataByKey(amID, "MusicBrainz", dbID)
    
def extraAddsForMusic(mdbmaps, chartType):
    if chartType == "Music":
        addAllMusic(mdbmaps, "Music", "9c8834ac0a7a7aaa66010ded94db1db6", "mn0000193444")   ### Cassius
        addDiscogs(mdbmaps, "Music", "9c8834ac0a7a7aaa66010ded94db1db6", "3358")    ### Cassius
        ### Feeling For You (Mix)  ,  Feeling For You  ,  09 La Mouche

        addAllMusic(mdbmaps, "Music", "39760181c887a11d9b31ca67d29b1958", "mn0000413000")   ### Diddy
        addDiscogs(mdbmaps, "Music", "39760181c887a11d9b31ca67d29b1958", "110844")    ### Diddy
        ### 

        addAllMusic(mdbmaps, "Music", "d897ffc65b3d7b934ce567e7c4a1e708", "mn0000311569")   ### Gabrielle
        addDiscogs(mdbmaps, "Music", "d897ffc65b3d7b934ce567e7c4a1e708", "5393")    ### Gabrielle
        ### Sunshine  ,  Gabrielle -  8. Falling  ,  Gabrielle -  2. Rise  ,  Gabrielle -  1. Sunshine  ,  Gabrielle -  5. 5 O'clock  ,  03 When a woman  ,  10. Independence day  ,  Gabrielle -  6. Should I stay  ,  11 Gonna Get Better  ,  Gabrielle -  7. Over you

        addAllMusic(mdbmaps, "Music", "ebfb692262c759c4c081824706045c4e", "mn0000111898")   ### <NAME>
        addDiscogs(mdbmaps, "Music", "ebfb692262c759c4c081824706045c4e", "257273")    ### <NAME>
        ### 

        addAllMusic(mdbmaps, "Music", "45a5064d057441062ea4e21d7042ca7a", "mn0000885569")   ### <NAME>
        addDiscogs(mdbmaps, "Music", "45a5064d057441062ea4e21d7042ca7a", "257663")    ### <NAME>
        ### The Michael Bublé Collection

        addAllMusic(mdbmaps, "Music", "96ea75ddbcf0a701a81c3d554cdb8976", "mn0000621112")   ### Mya
        addDiscogs(mdbmaps, "Music", "96ea75ddbcf0a701a81c3d554cdb8976", "28738")    ### Mya
        ### 

        addAllMusic(mdbmaps, "Music", "d70299388282bcc4e686cda2e0ee4027", "mn0000885216")   ### <NAME>
        addDiscogs(mdbmaps, "Music", "d70299388282bcc4e686cda2e0ee4027", "914802")    ### <NAME>
        ### 

        addAllMusic(mdbmaps, "Music", "acc2fb66c521367b1c6f74eabc6dbeb2", "mn0000017748")   ### Patti LaBelle
        addDiscogs(mdbmaps, "Music", "acc2fb66c521367b1c6f74eabc6dbeb2", "3474")    ### Patti LaBelle
        ### 

        addAllMusic(mdbmaps, "Music", "7af9e36da60b5f23852a8f43bb025a1f", "mn0000408736")   ### Real McCoy
        addDiscogs(mdbmaps, "Music", "7af9e36da60b5f23852a8f43bb025a1f", "17852")    ### Real McCoy
        ### Platinum & Gold Collection  ,  01 Real McCoy-- If You Should Eve  ,  01 Real McCoy-- Ooh Boy I Love Yo

        addAllMusic(mdbmaps, "Music", "e3bb0efde0825f1c46a6bc0670611c3a", "mn0002892368")   ### Stallion
        addDiscogs(mdbmaps, "Music", "e3bb0efde0825f1c46a6bc0670611c3a", "3184596")    ### Stallion
        ### 

        mdbmaps[chartType].save()
from multiArtist import multiartist
mularts  = multiartist(cutoff=0.9, discdata=None, exact=False)
knownMultiArtists = getFile("../multiartist/knownMultiArtists.yaml")
print(len(knownMultiArtists))
print(len(mdbmaps[chartType].getArtists()))
for idx,artistName in mdbmaps[chartType].getArtists().items():
    N = mularts.getArtistNames(artistName)
    if len(N) > 1:
        result = {subName: mdbmaps[chartType].isKnownByName(subName) for subName in N}
        if not any(result.values()):
            knownMultiArtists.append(artistName)
            #print(result)
        #print("ignores.append(\'{0}\')".format(artistName))

print(len(knownMultiArtists))
knownMultiArtists = list(set(knownMultiArtists))
print(len(knownMultiArtists))
        
saveFile(idata = sorted(knownMultiArtists), ifile="../multiartist/knownMultiArtists.yaml")
analyzeRenames(manDB, dbRenames)
toget
manDB = masterArtistNameDB("main", init=True)
manDB.forceReload(getFile("main.yaml"))
#manDB.addRenames(getFile("relDBRenames2.yaml"))
manDB.checkForRecursives()
manDB = masterArtistNameDB("main", init=True)
manDB.forceReload(getFile("main.yaml"))
#manDB.addRenames(getFile("relDBRenames2.yaml"))
manDB.checkForRecursives()
manDB.save()
manDB = masterArtistNameDB("main", init=False)
saveFile(idata=manDB.dbRenames, ifile="main.yaml")
removeDBRenames(mdbmaps, chartType, dbRenames)
for minAlbums,minAlbumsData in getIterItems().items():
    print("="*10,minAlbums)
    maxAlbums = minAlbumsData["Max"]
    maxValues = minAlbumsData["Vals"]
    for i in range(10):
        toMatch   = mdbmc.getDataToMatch(chartType, maxValues=maxValues, maxAlbums=maxAlbums, minAlbums=minAlbums, ignores=[])
        if len(toMatch[chartType]) == 0:
            break
        matchItAll(mdbmaps, getThresholds(minAlbums), toMatch=toMatch)
        
reMatch()
print("Done.\n\n")
toMatch   = mdbmc.getDataToMatch(chartType, maxValues=10000, maxAlbums=100000, minAlbums=0, useKnown=False, dbMatches=0, ignores=[])
toMatch<jupyter_output><empty_output><jupyter_text>Find DB Renames<jupyter_code>for idx,artistName in mdbmap.getArtists().items():
    rename = manDB.renamed(artistName)
    if artistName != rename:
        print('\t',artistName,'\t',rename)
    mdbmap.removeArtistByName(artistName)
    mdbmap.removeArtistByName(rename)
mdbmap.save()
from difflib import SequenceMatcher

ignores     = [] #getFile("chartIgnores.yaml")
dbRenames   = {} #getFile("relDBRenames2.yaml")

toget = {}

for primaryKey,artistName in mdbmap.getArtists().items():
    primaryArtistName = artistName
    artistData = mdbmap.getArtistDataByKey(primaryKey)
    for db,dbID in artistData.getDict().items():
        if db in ["DatPiff", "MetalStorm", "LastFM", "CDandLP"]:
            continue
        if dbID is not None:
            secondaryArtistName = maindb.getArtistDBNameFromID(db, dbID)
            if secondaryArtistName is None:
                if toget.get(db) is None:
                    toget[db] = []
                toget[db].append(primaryArtistName)
                continue
            if secondaryArtistName in ignores:
                continue
            s = SequenceMatcher(None, primaryArtistName, secondaryArtistName)
            ratio = s.ratio()
            maxRatio = 1.0
            if ratio >= maxRatio:
                continue
                
            if ratio >= maxRatio-0.2:
                if dbRenames.get(secondaryArtistName) is not None:
                    if dbRenames[secondaryArtistName] == primaryArtistName:
                        continue
                    else:
                        raise ValueError("Multi Values For {0}: [{1} , {2}]".format(secondaryArtistName, primaryArtistName, dbRenames[secondaryArtistName]))
                dbRenames[secondaryArtistName] = primaryArtistName
                print("[{0: <30} {1: <4} {2: >30}] \t --> ({3}) {4} / {5} ({6})".format(primaryArtistName,round(ratio,2),secondaryArtistName,"Music",primaryKey,dbID,db))
                
            
print("Done.")
mdbmap.addArtistByKey("5b06cf69aa2805be66a451f1841b76e0", "MusicBrainz", None)
mdbmap.addArtistByKey("1b8c61e7d426f75a8b1053d2b86c4d70", "Discogs", '251769')
mdbmap.addArtistByKey("d897ffc65b3d7b934ce567e7c4a1e708", "MusicBrainz", None)
mdbmap.addArtistByKey("681fdc0b6f5a02a25736afbdc18fa3df", "Discogs", "254198")
mdbmap.addArtistByKey("a79dcb1eb29bcfedfb7884cad2e9dc13", "Discogs", "6270")
mdbmap.save()
analyzeRenames(dbRenames)
manDB = masterArtistNameDB("main", init=False)
saveFile(idata=manDB.dbRenames, ifile="main.yaml")
manDB = masterArtistNameDB("main", init=True)
manDB.forceReload(getFile("main.yaml"))
#manDB.addRenames(getFile("relDBRenames2.yaml"))
manDB.checkForRecursives()
if True:
    manDB.save()
    manDB = masterArtistNameDB("main", init=False)
    saveFile(idata=manDB.dbRenames, ifile="main.yaml")
#[Supercharger                   0.92                   SuperCharger] --> None <-> 554456
mdbmap.addArtistDataByName("Supercharger", "Discogs", "554456")
mdbmap.addArtistDataByName("Supercharger", "AllMusic", "0000751305")
mdbmap.addArtistDataByName("Supercharger", "LastFM", "18980775548")


#[The Limeliters                 0.93               The Limelighters] --> None <-> 946411
mdbmap.addArtistDataByName("The Limeliters", "Discogs", "946411")
mdbmap.addArtistDataByName("The Limeliters", "AllMusic", "0000093533")
mdbmap.addArtistDataByName("The Limeliters", "LastFM", "62930540551")


#[Thee Headcoats                 0.93               Thee Headcoatees] --> None <-> 257206
mdbmap.addArtistDataByName("Thee Headcoats", "Discogs", "257206")
mdbmap.addArtistDataByName("Thee Headcoats", "AllMusic", "0000490296")
mdbmap.addArtistDataByName("Thee Headcoats", "LastFM", "72772112489")


#[Twista                         0.91                          Twist] --> None <-> 133869
mdbmap.addArtistDataByName("Twista", "Discogs", "133869")
mdbmap.addArtistDataByName("Twista", "AllMusic", "0000200679")
mdbmap.addArtistDataByName("Twista", "LastFM", "91542820655")


#[East Of Eden                   0.92                   East of Eden] --> None <-> 305975
mdbmap.addArtistDataByName("East Of Eden", "Discogs", "305975")
mdbmap.addArtistDataByName("East Of Eden", "AllMusic", "0000134333")
#mdbmap.removeArtistByName("NF")
mdbmap.save()
#mdbmap.getArtistDataByName("nf")
toget
#maindb.getArtistDBIDFromName("MusicBrainz", "<NAME>")
#maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/2258c0cf-77d7-4ddd-9440-f36e3c7a5e22")
#mdbmap.addArtistDataByName("<NAME>", "MusicBrainz", "320713839944087642428053781910236510708")
#mdbmap.getArtistDataByName("<NAME>")
#mdbmap.addArtistDataByName("<NAME>", "Discogs", "284554")
#mdbmap.addArtistDataByName("<NAME>", "AllMusic", "0000027864")
#mdbmap.save()
#del dbRenamesAdd["nf"]
del dbRenamesAdd["<NAME>"]
saveFile(ifile="../discogs/dbrenames.yaml", idata=dbRenamesAdd)<jupyter_output><empty_output><jupyter_text>Load My Music DataFrame<jupyter_code>def getArtistAlbums(artistName):
    from listUtils import getFlatList
    artistAlbumsDict = mmb.getArtistAlbumsByArtist(artistName)
    artistAlbumsByType = {}
    for albumType,albumsInfo in artistAlbumsDict.items():
        artistAlbumsByType[albumType] = getFlatList(albumsInfo.getAlbums().values())
        
    artistAlbums = []
    albumTypes = ["BoxSet", "Bootleg", "Match", "UnMatched"]
    for albumType in albumTypes:
        if isinstance(artistAlbumsByType.get(albumType), list):
            artistAlbums += [artistAlbumsByType[albumType]]
    return getFlatList(artistAlbums)


def getArtistSongs(artistName):
    from listUtils import getFlatList
    artistAlbumsDict = mmb.getArtistAlbumsByArtist(artistName)
    artistAlbumsByType = {}
    for albumType,albumsInfo in artistAlbumsDict.items():
        artistAlbumsByType[albumType] = getFlatList(albumsInfo.getAlbums().values())
        
    artistAlbums = []
    albumTypes = ["Random"]
    for albumType in albumTypes:
        if isinstance(artistAlbumsByType.get(albumType), list):
            artistAlbums += [artistAlbumsByType[albumType]]
    return getFlatList(artistAlbums)


def getMusicDF(mdbmap):
    mdf = mdbmap.getDF().T
    cols = mdf.columns

    mdfAlbums = DataFrame(Series({primaryKey: len(getArtistAlbums(primaryKey[0])) for primaryKey in mdbmap.getArtists()}))
    mdfAlbums.columns = ["Albums"]

    mdfSongs = DataFrame(Series({primaryKey: len(getArtistSongs(primaryKey[0])) for primaryKey in mdbmap.getArtists()}))
    mdfSongs.columns = ["Songs"]

    mdfMatches = DataFrame(Series({idx: len([x for x in row.values if x is not None]) for idx,row in mdf.iterrows()}))
    mdfMatches.columns = ["DBMatches"]

    mdfmapDF = mdf.join(mdfAlbums).join(mdfSongs).join(mdfMatches)
    mdfmapDF = mdfmapDF[["Albums"] + ["Songs"] + list(cols) + ["DBMatches"]]
    
    mdfmapDF = mdfmapDF.sort_values(by="Albums", ascending=False)
    
    return mdfmapDF
def addDiscogs(mdbmap, amID, dbID):
    if dbID is None:
        return
    mdbmap.addArtistDataByKey(amID, "Discogs", dbID)

def addAllMusic(mdbmap, amID, dbID):
    if dbID is None:
        return
    dbID = dbID[2:]
    mdbmap.addArtistDataByKey(amID, "AllMusic", dbID)

def addMusicBrainz(mdbmap, amID, url):
    if dbID is None:
        return
    dbID = maindb.getArtistDBIDFromUtil("MusicBrainz", url)
    mdbmap.addArtistDataByKey(amID, "MusicBrainz", dbID)
for artistName in unknownArtists.keys():
    idx = mdbmap.getPrimaryKey(artistName=artistName, artistID=None)
    print("mdbmap.addArtistByKey(\"{0}\", artistName=\"{1}\", artistID=None)".format(idx, artistName))
    print("add{0}(mdbmap, {1}, {2})   ### {3}".format("AllMusic", "\"{0}\"".format(idx), "\"{0}\"".format(""), artistName))    
    print("add{0}(mdbmap, {1}, {2})   ### {3}".format("Discogs", "\"{0}\"".format(idx), "\"{0}\"".format(""), artistName))
    print("\n")
mdbmap.addArtistByKey("34fde707c21e5bd8a91c3566daf2919c", artistName="InHuman", artistID=None)
addAllMusic(mdbmap, "34fde707c21e5bd8a91c3566daf2919c", "mn0000090299")   ### InHuman
addDiscogs(mdbmap, "34fde707c21e5bd8a91c3566daf2919c", "987955")   ### InHuman


mdbmap.addArtistByKey("<KEY>", artistName="República", artistID=None)
addAllMusic(mdbmap, "<KEY>", "mn0000886648")   ### República
addDiscogs(mdbmap, "affbc7f573df09ee07b3d98de5caed24", "11450")   ### República
mdbmap.save()
mdfmapDF = getMusicDF(mdbmap)
#mDBDF = mdfmapDF[mdfmapDF["DBMatches"] < 3]
mdbmap.addArtistByName("<NAME>")
mdbmap.addArtistDataByName("<NAME>", "AllMusic", "0000805740")
mdbmap.addArtistDataByName("<NAME>", "Discogs", "375279")
mdbmap.save()
mdbmap.addArtistByName("<NAME>")
mdbmap.addArtistDataByName("<NAME>", "AllMusic", "0000597875")
mdbmap.addArtistDataByName("<NAME>", "Discogs", "36543")
mdbmap.save()
kwargs    = {'numArtistName': 2, 'artistNameCutoff': 0.9, 'artistAlbumCutoff': 0.9, 'numArtistAlbums': 5, 'score': 5.75}

artistName = "<NAME>"
artistID = mdbmap.getPrimaryKey(artistName=artistName, artistID=None)
artistAlbums = maindb.getArtistDBAlbumsFromID("AllMusic", "0000805740")
len(artistAlbums)
mdbMatcher = matchDBArtist(maindb)
mdbMatcher.setArtistInfo(artistName, artistID, artistAlbums[:500])
mdbMatcher.setThresholds(matchNumArtistName=kwargs['numArtistName'], matchArtistNameCutoff=kwargs['artistNameCutoff'], 
                         matchArtistAlbumCutoff=kwargs['artistAlbumCutoff'], matchNumArtistAlbums=kwargs['numArtistAlbums'],
                         matchScore=kwargs['score'])
mcs    = mdbMatcher.findPotentialArtistAlbumMatches()
{db: mc.matchID for db,mc in mcs.items()}
mdbmap.addArtistDataByName("<NAME>", "MusicBrainz", "157547995485623313418745243608938386752")
mdbmap.save()
manDB = masterArtistNameDB()
manDB.createRenamesFromDB(mdbmap, dbRenames)
#artistNameDB["Dorrough"]
artistNameDB
dbRenames =getFile("masterDBRenames.yaml")
from difflib import SequenceMatcher
mdfmapDF = getMusicDF(mdbmap)
N = 0

dbName = "AllMusic"

for i,(idx,row) in enumerate(mdfmapDF[~mdfmapDF[dbName].isna()].iterrows()):
    if i < -1:
        continue
    artistName = idx[0]
    artistID   = idx[1]
    myAlbums = getArtistAlbums(artistName)
    mySongs  = getArtistAlbums(artistName)
    dbAlbums = getFlatList([maindb.getArtistDBAlbumsFromID(db,dbID) for db,dbID in row.to_dict().items() if dbID is not None and db not in ["DatPiff", "MetalStorm", "Albums", "Songs", "DBMatches"]])
    
    artistAlbums = dbAlbums + myAlbums + mySongs
    if len(artistAlbums) > 500:
        continue
    
    mdbMatcher = matchDBArtist(maindb)
    
    mdbMatcher.setThresholds(num=10, cutoff=0.8, matchReq=0.9)
    mdbMatcher.setArtistInfo(artistName, artistID, artistAlbums)
    
    mc = mdbMatcher.findPotentialArtistAlbumMatchesByDB(dbName)
    if mc.matchID is not None:
        dbArtistName = maindb.getArtistDBNameFromID(dbName, mc.matchID)
        s = SequenceMatcher(None, str(artistName), str(dbArtistName))
        if s.ratio() > 0.9:
            if mc.matchID != row.to_dict()[dbName]:
                print("#[{0: <30} {1: <4} {2: >30}] --> {3} / {4}".format(artistName,round(s.ratio(),2),dbArtistName,artistID,mc.matchID))
                print("mdbmap.addArtistData(\"{0}\", \"{1}\", \"{2}\", \"{3}\")".format(artistName, artistID, dbName, mc.matchID))
                print("")
                N += 1
        else:
            print("#[{0: <30} {1: <4} {2: >30}] --> {3} / {4}".format(artistName,round(s.ratio(),2),dbArtistName,artistID,mc.matchID))
            print("mdbmap.addArtistData(\"{0}\", \"{1}\", \"{2}\", None)".format(artistName, artistID, dbName))
            print("")
            N += 1
    else:
        continue
            
    if N > 11150:
        print(i)
        break

#[Pat Methany Group              0.94              Pat Metheny Group] --> 7cb12f5e2f9f9a1180e772e9707a40e4 / 0000178654
mdbmap.addArtistData("Pat Methany Group", "7cb12f5e2f9f9a1180e772e9707a40e4", "AllMusic", "0000178654")

mdbmap.save()
mdfmapDF = getMusicDF(mdbmap)
mDBDF = mdfmapDF[mdfmapDF["DBMatches"] < 3]
N = 0
for i,(idx,row) in enumerate(mDBDF.iterrows()):
    
    artistName = idx[0]
    artistID   = idx[1]
    myAlbums = getArtistAlbums(artistName)
    mySongs  = getArtistAlbums(artistName)
    dbAlbums = getFlatList([maindb.getArtistDBAlbumsFromID(db,dbID) for db,dbID in row.to_dict().items() if dbID is not None and db not in ["Albums", "Songs", "DBMatches"]])
    
    artistAlbums = dbAlbums + myAlbums + mySongs
    #print(i,'/',mDBDF.shape[0],'  \tN=',len(artistAlbums),'\t\t',idx)
    if len(artistAlbums) > 250:
        continue
    
    mdbMatcher = matchDBArtist(maindb)
    mdbMatcher.setArtistInfo(artistName, artistID, artistAlbums)
    mcs    = mdbMatcher.findPotentialArtistAlbumMatches()
    
    
    artistData = row.to_dict()
    known = []
    for db,mc in mcs.items():
        if mc.matchID is None:
            continue
        if artistData[db] is None:
            print("Adding {0} --> {1} ({2})".format(idx,db,mc.matchID))
            mdbmap.addArtistData(artistName, artistID, db, mc.matchID)
            known.append(db)
        else:
            if artistData[db] == mc.matchID:
                known.append(db)
                continue
            else:
                print("Found different IDs for {0} for DB {1}: {2}  <---->  {3}".format(idx, db, artistData[db], mc.matchID))

        
    #print('\t\t\t\t',known)
    if "MusicBrainz" not in known:
        print('\t\t\t\t',"mdbmap.addArtistData(\"{0}\", \"{1}\", 'MusicBrainz', maindb.dbdata[\"MusicBrainz\"][\"Utils\"].getArtistID(\"\")".format(artistName, artistID))
        continue
        
    N += 1
    if N >= 10:
        break
mdbmap.addArtistData("amazarashi", "6038fb09d5a38f594ea7e697bb34d912", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/105f5e66-9742-40e8-a1f2-bc9781f935e8"))
mdbmap.addArtistData("Sirenia", "1b7d00a447586a472e7a99ad069dd5eb", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/5fcd6db8-979c-42e3-99a6-fd8302f82a9d"))
mdbmap.addArtistData("No Use For A Name", "3effa76ba6ef83bd4071d94c6b047919", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/d7bd0fad-2f06-4936-89ad-60c5b6ada3c1"))
mdbmap.addArtistData("Celtic Thunder", "372b55783b7ffcf7bc2ed411dea5362e", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/9f176bd8-c032-4c39-aac8-fc663072a77e"))
mdbmap.addArtistData("Enchant", "67837239917008c31fa04cb035071ac8", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/a01bbd43-6ed3-4773-82f1-4c50908a6e58"))
mdbmap.addArtistData("<NAME>", "a2def3b0111283e384a376742315d93c", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/4ea47188-de59-4d11-9154-657749319a41"))
mdbmap.addArtistData("Pengshui", "a514f35d39011eab1cd235a3e357365a", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/43ba28aa-171e-4275-bf77-138d5e2bec07"))
mdbmap.addArtistData("<NAME>", "a82c7adf004fc08cc97373495259ce9f", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/98b1796d-a888-4a69-9c2a-86bfff3156f1"))
mdbmap.addArtistData("He Is Legend", "64770fa3527e87fa5ea7de09daed84ea", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/b9fb5447-7f95-4a6a-a157-afed2d7b9f4c"))
mdbmap.addArtistData("Endorfin.", "f60e48ee28dd2bb5975d8a1213fc5f6e", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/dcba8088-5874-46aa-8a1d-0f0f58ba7dcd"))
mdbmap.addArtistData("<NAME>", "2d45ef2d01464a825d6273875629b159", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/9ff4f432-5e01-489d-bb92-545ef1728503"))
mdbmap.addArtistData("Fruupp", "9a302161d5d9dfc193095c6c043cd048", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/2687b7e9-a7a0-40dc-96c2-659d4f3ab290"))
mdbmap.addArtistData("<NAME>", "576427c10e6d847ebe00dd7c625cf440", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/3ceb73ec-26e1-40fe-a2b8-3f3ed43a841e"))
mdbmap.addArtistData("<NAME>", "e0ae132bd51fc88c0581374d1ab51ff7", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/ecf3698d-f86a-4f9b-b462-36e509c7065c"))
mdbmap.addArtistData("<NAME>", "7297db56cc96e80010c302aeee5bb925", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/6d44a488-28e8-47dd-bed7-bbe09bd5a80b"))
mdbmap.addArtistData("<NAME>", "ba80c01158c84de1f8f521722c3c9773", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/71d58182-aa37-4c04-b21a-efe46ea0f221"))
mdbmap.addArtistData("This Mortal Coil", "9ce11709d6c643dee71d3a06590da1c4", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/60ef8ddb-7676-4056-a357-9a8c228d03a6"))
mdbmap.addArtistData("<NAME>", "2955a38a4a6fd1d9d71c19a1f16b2f97", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/d189d81a-e516-40e6-a10c-b8160742ff55"))
mdbmap.addArtistData("<NAME>", "7aaf61d830e7014a3496391b25281784", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/84c7cb93-0622-4b90-8d53-ee2b4650097f"))
mdbmap.addArtistData("Stormzy", "5916e565c90c564d6cf46566b58253dd", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/67fd5116-7a49-4abf-a34f-b08774128a7a"))
mdbmap.addArtistData("702", "b1eec33c726a60554bc78518d5f9b32c", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/4822d466-5c1f-4e04-80e9-e33e8295c26b"))
mdbmap.addArtistData("<NAME>", "093d6cc6733f63aa9faef9f66ed9247a", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/9da87628-95a0-4820-93c7-7b1609d9bce7"))
mdbmap.addArtistData("<NAME>", "bf3ae6e30f7029dd22944216f83fa1c7", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/12897e49-8af3-472f-86ab-412c2051956d"))
mdbmap.addArtistData("<NAME>", "78a49ae690e890c35052e1aa85cbff47", 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/80173caa-c6c0-4c66-8598-68a2c61d2415"))
mdbmap.addArtistData('<NAME>', '7524b89192ffd70026a81383f8d6e41a', "MusicBrainz", maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/09ff1fe8-d61c-4b98-bb82-18487c74d7b7"))
mdbmap.save()
mdbmap.getArtistData('<NAME>', '7524b89192ffd70026a81383f8d6e41a').show()
mdbmap.save()
mdbmap.getArtistData('<NAME>', '7524b89192ffd70026a81383f8d6e41a')<jupyter_output><empty_output><jupyter_text>My Music Database Map<jupyter_code>from difflib import SequenceMatcher
from pandas import Series, DataFrame, isna, isnull

def getDBDF(mdbmc, dbName, dbShort, keepNA=False):
    amdf = DataFrame(mdbmaps[dbName].getDF().T)
    print("Total Size: {0}".format(amdf.shape[0]))
    dbcols = amdf.columns

    matchData  =  mdbmc.getDBMatchData(dbName)
    mAlbDF = DataFrame(Series({primaryKey: len(albums) for primaryKey,albums in matchData.items()}))

    amdf = amdf.join(mAlbDF)
    cols = list(amdf.columns)
    cols[-1] = "{0}Albums".format(dbShort)
    amdf.columns = cols

    amdf.reset_index(inplace=True)
    columns = list(amdf.columns)
    columns[0] = "{0}ArtistName".format(dbShort)
    columns[1] = "{0}Key".format(dbShort)
    amdf.columns = columns
    if keepNA is False:
        amdf = amdf[~amdf[dbName].isna()]

    colOrder = columns[:2] + [columns[-1]] + list(dbcols)
    amdf = amdf[colOrder]

    amdf = amdf.sort_values("{0}Albums".format(dbShort), ascending=False)

    print(" Good Size: {0}".format(amdf.shape[0]))
    return amdf


def getMyDBDF(mdbmap, dbName, dbShort, keepNA=False):
    amdf = DataFrame(mdbmap.getDF().T)
    print("Total Size: {0}".format(amdf.shape[0]))
    dbcols = amdf.columns

    amdf.reset_index(inplace=True)
    columns = list(amdf.columns)
    columns[0] = "{0}ArtistName".format(dbShort)
    columns[1] = "{0}Key".format(dbShort)
    amdf.columns = columns
    if keepNA is False:
        amdf = amdf[~amdf[dbName].isna()]
    print(" Good Size: {0}".format(amdf.shape[0]))
    return amdf


def checkDBIDMatch(amdf, dbName, dbShort):
    wrong = amdf[amdf["{0}Key".format(dbShort)] != amdf[dbName]]
    idxs  = list(wrong.index)
    print("Found {0} wrongly assigned index".format(len(idxs)))
    return idxs
 
    
#####################################################################################################################
# Fill Matched DB Data From Overlapping Data In A Previously Matched DB
#####################################################################################################################
def mutualMatch(amdf, mdbmaps, primaryInfo, secondaryInfo, debug=False, test=True, ratioCut=0.9):
    primaryDBName  = primaryInfo[0]
    primaryDBShort = primaryInfo[1]
    primaryDBArtistName = "{0}ArtistName".format(primaryDBShort)
    primaryDBKey        = "{0}Key".format(primaryDBShort)

    secondaryDBName  = secondaryInfo[0]
    secondaryDBShort = secondaryInfo[1]
    
    print("Primary:")
    print("  primaryDBName:       {0}".format(primaryDBName))
    print("  primaryDBShort:      {0}".format(primaryDBShort))
    print("  primaryDBArtistName: {0}".format(primaryDBArtistName))
    print("  primaryDBKey:        {0}".format(primaryDBKey))
    print("Secondary:")
    print("  secondaryDBName:     {0}".format(secondaryDBName))
    print("  secondaryDBShort:    {0}".format(secondaryDBShort))


    debug = debug
    amrcdf = amdf[~amdf[secondaryDBName].isna()][[primaryDBArtistName, primaryDBKey, secondaryDBName]]
    for i,row in amrcdf.iterrows():

        ## Primary DB Key --> Secondary ID
        amDBName = row[primaryDBArtistName]
        amDBKey  = row[primaryDBKey]
        rcDBKey  = row[secondaryDBName]

        ## Secondary ID --> Secondary DB Key
        primKey  = mdbmaps[secondaryDBName].getPrimaryKeyFromID(rcDBKey)
        if primKey is None:
            continue
        rcDBName = primKey[0]
        rcDBKey  = primKey[1]


        s = SequenceMatcher(None, str(amDBName), str(rcDBName))

        if debug is True:
            print("{0: <30}{1: <30}{2: <30}{3}".format("", primaryDBName, secondaryDBName, "Match"))
            print("{0: <30}{1: <30}{2: <30}{3}".format("{0} ID   --> {1} ID".format(primaryDBShort, secondaryDBShort), amDBKey, rcDBKey, round(s.ratio(),2)))
            print("{0: <30}{1: <30}{2: <30}".format("{0} Name --> {1} Name".format(primaryDBShort, secondaryDBShort), amDBName, rcDBName))


        amDBData = mdbmaps[primaryDBName].getArtistData(amDBName, amDBKey)
        rcDBData = mdbmaps[secondaryDBName].getArtistData(rcDBName, rcDBKey)

        amRCDBKey = rcDBKey
        rcAMDBKey = rcDBData.getDBID(primaryDBName)
        if debug is True:
            print("{0: <30}{1: <30}{2: <30}".format("{0} ID   --> {1} ID".format(secondaryDBShort, primaryDBShort), str(rcAMDBKey), str(amRCDBKey)))    

        if s.ratio() > ratioCut:
            if rcAMDBKey != amDBKey:
                print("{0: <50}{1: <75}\t--->\t[{2}/{3}]".format("Setting {0} DBData For {1}".format(secondaryDBName,primaryDBName),"[{0: <30} {1: <4} {2: >30}]".format(amDBName,round(s.ratio(),2),rcDBName),amDBKey,rcDBKey))
                if test is True:
                    continue
                mdbmaps[secondaryDBName].addArtistData(rcDBName, rcDBKey, primaryDBName, amDBKey)

        if debug:
            print("\n")

    if test is True:
        return
    mdbmaps[secondaryDBName].save()
    
    
    
#####################################################################################################################
# Fill Fresh DB Data From A Previously Matched DB
#####################################################################################################################
def mutualMatchFromPreviousDBMatch(amdf, mdbmaps, primaryInfo, secondaryInfo, debug=False, test=True, ratioCut=0.9):

    primaryDBName  = primaryInfo[0]
    primaryDBShort = primaryInfo[1]
    primaryDBArtistName = "{0}ArtistName".format(primaryDBShort)
    primaryDBKey        = "{0}Key".format(primaryDBShort)

    secondaryDBName  = secondaryInfo[0]
    secondaryDBShort = secondaryInfo[1]
    
    print("Primary:")
    print("  primaryDBName:       {0}".format(primaryDBName))
    print("  primaryDBShort:      {0}".format(primaryDBShort))
    print("  primaryDBArtistName: {0}".format(primaryDBArtistName))
    print("  primaryDBKey:        {0}".format(primaryDBKey))
    print("Secondary:")
    print("  secondaryDBName:     {0}".format(secondaryDBName))
    print("  secondaryDBShort:    {0}".format(secondaryDBShort))




    amrcdf = amdf[~amdf[secondaryDBName].isna()][[primaryDBArtistName, primaryDBKey, secondaryDBName]]
    for i,row in amrcdf.iterrows():

        ## Primary DB Key --> Secondary ID
        amDBName = row[primaryDBArtistName]
        amDBKey  = row[primaryDBKey]
        rcDBKey  = row[secondaryDBName]

        if amDBKey is None:
            continue

        #if debug is True:
        #    print(amDBName,'\t',amDBKey,'\t',rcDBKey,'\t-->\t',end="")

        ## Secondary ID --> Secondary DB Key
        primKey = toMatchKeys.get(rcDBKey)
        #if debug is True:
        #    print(primKey)
        #primKey  = mdbmaps[secondaryDBName].getPrimaryKeyFromID(rcDBKey)

        if primKey is None:
            continue
        rcDBName = primKey[0]
        rcDBKey  = primKey[1]



        s = SequenceMatcher(None, str(amDBName), str(rcDBName))

        if debug is True:
            print("{0: <30}{1: <30}{2: <30}{3}".format("", primaryDBName, secondaryDBName, "Match"))
            print("{0: <30}{1: <30}{2: <30}{3}".format("{0} ID   --> {1} ID".format(primaryDBShort, secondaryDBShort), amDBKey, rcDBKey, round(s.ratio(),2)))
            print("{0: <30}{1: <30}{2: <30}".format("{0} Name --> {1} Name".format(primaryDBShort, secondaryDBShort), amDBName, rcDBName))

        continue
        if not mdbmaps[secondaryDBName].isKnown(rcDBName, rcDBKey):
            mdbmaps[secondaryDBName].addArtist(rcDBName, rcDBKey)
            mdbmaps[secondaryDBName].addArtistData(rcDBName, rcDBKey, secondaryDBName, rcDBKey)


        amDBData = mdbmaps[primaryDBName].getArtistData(amDBName, amDBKey)
        rcDBData = mdbmaps[secondaryDBName].getArtistData(rcDBName, rcDBKey)

        amRCDBKey = rcDBKey
        rcAMDBKey = rcDBData.getDBID(primaryDBName)
        if debug is True:
            print("{0: <30}{1: <30}{2: <30}".format("{0} ID   --> {1} ID".format(secondaryDBShort, primaryDBShort), str(rcAMDBKey), str(amRCDBKey)))    

        if s.ratio() > ratioCut:
            if rcAMDBKey != amDBKey:
                print("{0: <50}{1: <75}\t--->\t[{2}/{3}]".format("Setting {0} DBData For {1}".format(secondaryDBName,primaryDBName),"[{0: <30} {1: <4} {2: >30}]".format(amDBName,round(s.ratio(),2),rcDBName),amDBKey,rcDBKey))
                if test is True:
                    continue
                mdbmaps[secondaryDBName].addArtistData(rcDBName, rcDBKey, primaryDBName, amDBKey)


        if debug:
            print("\n")

    if test is False:
        mdbmaps[secondaryDBName].save()    
    
    

def isSame(x):
    name  = x.MyArtistName
    match = x["{0}ArtistName".format(dbShort)]
    if not all([name,match]):
        return None
    if any([isna(x) for x in [name,match]]):
        return None

    same = False
    if all([name,match]):
        if str(name) == str(match):
            same = True
    else:
        raise ValueError([name,match])
    return same

def ratio(x):
    name  = x.MyArtistName
    match = x["{0}ArtistName".format(dbShort)]
    same  = x.Same
    if same is None:
        return None
    else:
        if all([name,match]):
            s = SequenceMatcher(None, str(name), str(match))
            return s.ratio()
        return -1.0<jupyter_output><empty_output><jupyter_text>Load My Music DB<jupyter_code>#mdbmap.removeArtist("<NAME> & <NAME>")
#mdbmap.save()
mdbmap = musicDBMap("Music", init=False)<jupyter_output><empty_output><jupyter_text>Load DB Data<jupyter_code>dbDict = {}
dbDict["AllMusic"]      = "AM"
dbDict["Discogs"]       = "DC"
dbDict["MusicBrainz"]   = "MB"
dbDict["LastFM"]        = "LM"
dbDict["RockCorner"]    = "RC"
dbDict["MusicStack"]    = "MS"
dbDict["CDandLP"]       = "CL"
dbDict["AceBootlegs"]   = "AB"
dbDict["RateYourMusic"] = "RM"
dbDict
loadAll = False
if loadAll:
    mdbmaps = {}
    for db in maindb.getDBs():
        print("{0: <30}".format(db), end="")
        mdbmaps[db] = musicDBMap(db, init=False)
    #mdbmaps = {db: musicDBMap(db, init=False) for db in maindb.getDBs()}
else:
    mdbmaps={}
    mdbmaps["AllMusic"]      = musicDBMap("AllMusic", init=False)
    mdbmaps["MusicBrainz"]   = musicDBMap("MusicBrainz", init=False)
    mdbmaps["Discogs"]       = musicDBMap("Discogs", init=False)
    if False:
        mdbmaps = {}
        mdbmaps["AllMusic"]      = musicDBMap("AllMusic", init=False)
        mdbmaps["MusicBrainz"]   = musicDBMap("MusicBrainz", init=False)
        mdbmaps["RockCorner"]    = musicDBMap("RockCorner", init=False)
        mdbmaps["MusicStack"]    = musicDBMap("MusicStack", init=False)
        mdbmaps["CDandLP"]       = musicDBMap("CDandLP", init=False)
        mdbmaps["AceBootlegs"]   = musicDBMap("AceBootlegs", init=False)
        mdbmaps["RateYourMusic"] = musicDBMap("RateYourMusic", init=False)<jupyter_output><empty_output><jupyter_text>Load Main DB and Match Class<jupyter_code>maindb = mainDB(mdb=None, create=False, debug=False)
maindb.loadDBDataMap()
mdbmc = masterDBMatchClass(maindb, mdbmaps)<jupyter_output><empty_output><jupyter_text>Matching Area<jupyter_code>mdbmap  = musicDBMap("Music", init=False)
dbName  = "Discogs"
dbShort = "DC"
getDBDFs = True
if getDBDFs:
    mdfs = {dbNameKey: getDBDF(mdbmc, dbNameKey, dbDict[dbNameKey]) for dbNameKey in mdbmaps.keys()}
    for dbNameKey in mdfs.keys():
        idxs = checkDBIDMatch(mdfs[dbNameKey], dbNameKey, dbDict[dbNameKey])
        print("")
def getMyDF(mdbmap, dbName):
    myamdf = DataFrame(mdbmap.getDF().T[dbName])
    myamdf.reset_index(inplace=True)
    myamdf.columns = ["MyArtistName", "MyKey", "DBID"]
    myamdf = myamdf[~myamdf["DBID"].isna()]
    return myamdf

def getMyDFMap(mdbmap, dbName):
    myamdf = mdbmap.getDF().T
    myamdf = myamdf[~myamdf[dbName].isna()]
    return myamdf
myamdf = getMyDF(mdbmap, dbName)

amdf = mdfs[dbName][["{0}ArtistName".format(dbShort), "{0}Key".format(dbShort), "{0}".format(dbName)]].copy(deep=True)
cols = list(amdf.columns)
cols[-1] = "DBID"
amdf.columns = cols

from pandas import merge
mgdf = merge(myamdf, amdf, on=['DBID'], how='inner')
dbShort = dbDict[dbName]
print(dbName,dbShort)
#mgdf[mgdf["AMKey"].isna()]
print(mgdf.shape)<jupyter_output><empty_output><jupyter_text>Check for mutual names<jupyter_code>dbRenames = getFile("dbRenames.yaml")
mgdf["{0}ArtistName".format(dbShort)] = mgdf["{0}ArtistName".format(dbShort)].apply(lambda x: dbRenames.get(x) if dbRenames.get(x) else x)
mgdf["Same"]  = mgdf.apply(isSame, axis=1)
mgdf["Ratio"] = mgdf.apply(ratio, axis=1)
rDF = mgdf[((mgdf['Same'] == False) & (mgdf['Ratio'] > 0.0) & (mgdf['Ratio'] < 1.7))][["MyArtistName", "{0}ArtistName".format(dbShort)]]
print("Size ==> {0}".format(mgdf[(mgdf['Same'] == False)].shape[0]))
print("-"*20)
for i,row in rDF.iterrows():
    myName = row.MyArtistName
    amName = row["{0}ArtistName".format(dbShort)]
    #print("mdbmap.addArtistData(\"{0}\", mdbmap.getHash(\"{1}\"), 'MusicBrainz', None)".format(myName, myName))
    #print("mdbmap.addArtistData(\"{0}\", mdbmap.getHash(\"{1}\"), 'MusicBrainz', maindb.dbdata[\"MusicBrainz\"][\"Utils\"].getArtistID(\"\"))".format(myName, myName))
    #continue
    print("{0}: {1}".format(amName, myName))
    if myName.startswith("The ") and not amName.startswith("The "):
        print("The The {0}: {1}".format(amName, myName))
artistName = "<NAME>"
row = mgdf[mgdf["MyArtistName"] == artistName]
myKey = row.MyKey.values[0]
print("Artist --> {0}".format(artistName))
print("MyKey  --> {0}".format(myKey))
row
mdbmap.addArtistData(artistName, myKey, 'Discogs', '897831')
mdbmap.save()
mdbmap.getArtistData("Jay-Z").show()<jupyter_output><empty_output><jupyter_text>Mutual Matches<jupyter_code>mydf = mdbmap.getDF()
mydf.sum(axis=0)
maindb.dbdata["Discogs"]["Utils"].getArtistID()
#mdbmap.removeArtist("<NAME>", "056cf7c048a98f80e77e6cef4a129e6c")
mdbmap.removeArtist("Wally", "8789b45de32801ad220f59e66d29f3e9")
mdbmap.save()
myamdf[~myamdf["AllMusic"].isna()]  #[[primaryDBArtistName, primaryDBKey, secondaryDBName]]
mutualMatch(myamdf, mdbmaps, ("Discogs", "DC"), ("AllMusic", "AM"), debug=False, test=True)
allmusicDF = mdbmap.getDF().T["MusicBrainz"]
toget = []
print("{0: <45}{1: <50}{2: <50}{3}".format("ID", "My Name","DB Name","Ratio"))
for idx,item in allmusicDF[~allmusicDF.isna()].iteritems():
    #artistName = maindb.getArtistDBNameFromID("MusicBrainz", item)
    artistName = maindb.dbdatamap["MusicBrainz"].getArtistNameFromID(item)
    if artistName is None:
        artistName = ""
        toget.append(idx[0])
    primaryKey = (artistName,item)
    if idx[0] != artistName:
        s = SequenceMatcher(None, str(idx[0]), str(artistName))
        ratio = int(100*s.ratio())
        if ratio < 80:
            print("{0: <45}{1: <50}{2: <50}{3}".format(item, idx[0],artistName,ratio))
mdbmap.addArtistData("Project Z", mdbmap.getHash("Project Z"), "MusicBrainz", maindb.dbdata["Discogs"]["Utils"].getArtistID("https://musicbrainz.org/artist/3837e81c-d3bb-4451-9206-a1f0292b6e69"))
mdbmap.save()
a = maindb.dbdatamap["AllMusic"].getArtists()
for x in a:
    if "Berlioz" in x:
        print(x)
maindb.dbdatamap["MusicBrainz"].getArtistIDFromName("<NAME>")
#####################################################################################################################
# Fill Matched DB Data From Overlapping Data In A Previously Matched DB
#####################################################################################################################
mydf = mdbmap.getDF().T
primaryInfo   = ("Discogs", "DC")
secondaryInfo = ("LastFM", "FM")
debug=False
test=True
ratioCut=0.9


primaryDBName  = primaryInfo[0]
primaryDBShort = primaryInfo[1]
primaryDBArtistName = "{0}ArtistName".format(primaryDBShort)
primaryDBKey        = "{0}Key".format(primaryDBShort)

secondaryDBName  = secondaryInfo[0]
secondaryDBShort = secondaryInfo[1]

print("Primary:")
print("  primaryDBName:       {0}".format(primaryDBName))
print("  primaryDBShort:      {0}".format(primaryDBShort))
print("  primaryDBArtistName: {0}".format(primaryDBArtistName))
print("  primaryDBKey:        {0}".format(primaryDBKey))
print("Secondary:")
print("  secondaryDBName:     {0}".format(secondaryDBName))
print("  secondaryDBShort:    {0}".format(secondaryDBShort))

debug = debug

pMyDF = mydf[~mydf[primaryDBName].isna()].copy(deep=True)


for i,(idx,row) in enumerate(pMyDF.iterrows()):
    if i < iMax:
        continue
    
    myArtistName = idx[0]
    myArtistID   = idx[1]
    if myArtistID in myignores:
        continue
    
    primDBKey = row[primaryDBName]
    secoDBKey = row[secondaryDBName]
    if not all([primDBKey, secoDBKey]):
        continue
    
    #print("MyID: {0}\t\tMyName: {1}".format(myArtistID,myArtistName))
    #print("Primary: {0} \t\tSecondary: {1}".format(primDBKey,secDBKey))
    
    primDBInfo = mdbmaps[primaryDBName].getPrimaryKeyFromID(primDBKey)
    secoDBInfo = mdbmaps[secondaryDBName].getPrimaryKeyFromID(secoDBKey)
    
    #print("Primary Info:   {0}".format(primDBInfo))
    #print("Secondary Info: {0}".format(secoDBInfo))
    
    try:
        primDBData = mdbmaps[primaryDBName].getArtistData(primDBInfo[0], primDBInfo[1])
    except:
        iMax = i
        #print(i,idx,row)
        print("toget.append(\"{0}\")".format(myArtistName))
        continue
        raise ValueError("Primary DB Error: {0}".format((primDBKey,primDBInfo)))
        
    try:
        secoDBData = mdbmaps[secondaryDBName].getArtistData(secoDBInfo[0], secoDBInfo[1])
    except:
        iMax = i
        #print(i,idx,row)
        print("toget.append(\"{0}\")".format(myArtistName))
        continue
        raise ValueError("Secondary DB Error: {0}".format((secoDBKey,secoDBInfo)))
    
    
    continue
    
    primDBIDs = primDBData.getDict()
    secoDBIDs = secoDBData.getDict()

    
    primDBID_primID = primDBIDs[primaryDBName]
    primDBID_secoID = primDBIDs[secondaryDBName]
    
    secoDBID_primID = secoDBIDs[primaryDBName]
    secoDBID_secoID = secoDBIDs[secondaryDBName]

    if all([primDBKey, primDBID_primID, primDBID_secoID]):
        print("{0: <30}{1: <30}{2}".format(primDBKey, primDBID_primID, primDBID_secoID))
    else:
        print("{0: <30}".format(primDBKey), end="")
        if primDBID_primID is None:
            print("{0: <30}".format("None"), end="")
        else:
            print("{0: <30}".format(primDBID_primID), end="")
        if primDBID_secoID is None:
            print("{0: <30}".format("None"), end="")
        else:
            print("{0: <30}".format(primDBID_secoID), end="")
         
    print('\n\n')   
    continue
            
            
    if all([secoDBKey, secoDBID_primID, secoDBID_secoID]):
        print("{0: <30}{1: <30}{2}".format(secoDBKey, secoDBID_primID, secoDBID_secoID))
    
    if primDBKey == primDBID_primID:
        print("\tPassed Primary   <-> Primary   Match")
    else:
        print("\tFailed Primary   <-> Primary   Match")
        print("\tMy PrimaryDBKey: {0}".format(primDBKey))
        print("\tDB PrimaryDBKey: {0}".format(primDBID_primID))
        1/0
    
    print('\n\n')
    continue
        
    if secoDBKey == primDBID_secoID:
        print("\tPassed Primary   <-> Secondary Match")
    if primDBKey == secoDBID_primID:
        print("\tPassed Secondary <-> Primary   Match")
    if secoDBKey == secoDBID_secoID:
        print("\tPassed Secondary <-> Secondary Match")
        
    
    1/0
    
    ## Secondary ID --> Secondary DB Key
    primKey  = mdbmaps[secondaryDBName].getPrimaryKeyFromID(rcDBKey)
    if primKey is None:
        continue
    rcDBName = primKey[0]
    rcDBKey  = primKey[1]


    s = SequenceMatcher(None, str(amDBName), str(rcDBName))

    if debug is True:
        print("{0: <30}{1: <30}{2: <30}{3}".format("", primaryDBName, secondaryDBName, "Match"))
        print("{0: <30}{1: <30}{2: <30}{3}".format("{0} ID   --> {1} ID".format(primaryDBShort, secondaryDBShort), amDBKey, rcDBKey, round(s.ratio(),2)))
        print("{0: <30}{1: <30}{2: <30}".format("{0} Name --> {1} Name".format(primaryDBShort, secondaryDBShort), amDBName, rcDBName))


    amDBData = mdbmaps[primaryDBName].getArtistData(amDBName, amDBKey)
    rcDBData = mdbmaps[secondaryDBName].getArtistData(rcDBName, rcDBKey)

    amRCDBKey = rcDBKey
    rcAMDBKey = rcDBData.getDBID(primaryDBName)
    if debug is True:
        print("{0: <30}{1: <30}{2: <30}".format("{0} ID   --> {1} ID".format(secondaryDBShort, primaryDBShort), str(rcAMDBKey), str(amRCDBKey)))    

    if s.ratio() > ratioCut:
        if rcAMDBKey != amDBKey:
            print("{0: <50}{1: <75}\t--->\t[{2}/{3}]".format("Setting {0} DBData For {1}".format(secondaryDBName,primaryDBName),"[{0: <30} {1: <4} {2: >30}]".format(amDBName,round(s.ratio(),2),rcDBName),amDBKey,rcDBKey))
            if test is True:
                continue
            mdbmaps[secondaryDBName].addArtistData(rcDBName, rcDBKey, primaryDBName, amDBKey)

    if debug:
        print("\n")

if test is True:
    return
mdbmaps[secondaryDBName].save()
toget = []
toget
myignores = []
toget
#mdbmap.addArtistData('-123 min.', '8311291b86c3d52634444dfb330f42b3', "AllMusic", '0000422575')
#mdbmap.save()
mdbmap.removeArtist("The English Beat", mdbmap.getHash("The English Beat"))
mdbmap.save()
maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("c945ec66-a07c-3986-86a7-6db575d99740")
mdbmap.removeArtist("Jo<NAME>asco and The El Molino Band")
mdbmap.save()
mdbmap.addArtistData("Lil' Boosie", mdbmap.getHash("Lil' Boosie"), 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/8ba17cf6-bec2-4ae4-9820-b1cda47adc08"))
mdbmap.save()
maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/769b978f-8b3e-42cc-8a3b-9c1121c5dc0f")
maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("769b978f-8b3e-42cc-8a3b-9c1121c5dc0f")
mdbmap.addArtistData("Tone-Loc", mdbmap.getHash("Tone-Loc"), 'MusicBrainz', maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/16da4997-78e1-4837-85f6-9247d10cde1f"))
mdbmap.save()
mdbmap.save()
for primaryKey,dbID in mdf.T["AllMusic"].to_dict():
    artistName = primaryKey[0]
    mdbmaps[]
### Master DB code
%load_ext autoreload
%autoreload
maindb = mainDB(mdb=None, create=False, debug=False)
maindb.loadDBDataMap()
mdbmap = musicDBMap("Music", init=False)
if True:
    mdb = myMusicDBMap("Music", debug=False)
    #mdb.getFullDBData()
    #mdb.getKnownDBData()
    maindb = mainDB(mdb=None, create=False, debug=True)
    maindb.loadDBDataMap()
    dbdata = maindb.dbdata
    #maindb.setDBFull() ## Do this to recreate everything
    #maindb.setDBKnown()
    _, _ = clock("Last Run")<jupyter_output><empty_output><jupyter_text>Find All My Music and What's Known/Unknown  Find My Music<jupyter_code>mdbmap.stats()
mdbmap.add("Wälly", mdbmap.getHash("Wälly"), "MusicBrainz", maindb.dbdata["MusicBrainz"]["Utils"].getArtistID("https://musicbrainz.org/artist/3b6c3af4-62b3-412e-9445-ae6b7ede2b86"))
#mdb.add("<NAME>", "Discogs", "1233607")
mdb.save()<jupyter_output><empty_output><jupyter_text>Find Mutual Entries<jupyter_code>def getMatchDataForArtist(artistName, mdbmap):
    artistAlbums = getArtistAlbums(artistName)
    _, artistID  = mdbmap.getPrimaryKeyFromName(artistName)    
    return artistName, artistID, artistAlbums
mdfAlbums = {primaryKey: len(getArtistAlbums(primaryKey[0])) for primaryKey in mdbmap.getArtists()}
toMatch = list(rymDF[rymDF.isna()].index)
len(toMatch)
mdbMatcher = matchDBArtist(maindb)
matchResults = {}
for primaryKey in toMatch:
    print(primaryKey)
    artistName   = primaryKey[0]
    artistID     = primaryKey[1]
    artistAlbums = getArtistAlbums(artistName)
    mdbMatcher.setArtistInfo(artistName, artistID, artistAlbums)
    mc = mdbMatcher.findPotentialArtistAlbumMatchesByDB('LastFM')
    matchResults[primaryKey] = mc
mydf = mdbmap.getDF().T
mydf[~mydf.AllMusic.isna() & ~mydf.MusicBrainz.isna()]
newDB = "LastFM"
mydf = mdbmap.getDF().T
mdbMatcher = matchDBArtist(maindb)

results = {}
saves   = 0
nDF     = mydf.shape[0]
for i,(idx,row) in enumerate(mydf.iterrows()):
    if i <= 4699:
        continue
    myArtistName      = idx[0]
    myArtistID        = idx[1]
    

    dbMatchIDs    = row.to_dict()
    dbMatchNames  = {}
    dbMatchAlbums = {}
    
    allAlbums = []

    toMatch = []
    for dbName,dbMatchID in dbMatchIDs.items():
        if dbName in ["DatPiff", "MetalStorm"]:
            continue
        if dbMatchID is None:
            dbMatchNames[dbName]  = None
            dbMatchAlbums[dbName] = []
            toMatch.append(dbName)
            continue
        dbMatchNames[dbName] = maindb.getArtistDBNameFromID(dbName, dbMatchID)
        if dbMatchNames[dbName] is None:
            print(idx,'\t',dbName)
            dbMatchAlbums[dbName] = []
        else:
            dbMatchAlbums[dbName] = maindb.getArtistDBAlbumsFromID(dbName, dbMatchID)
            allAlbums += dbMatchAlbums[dbName]
            
    allAlbums = list(set(allAlbums))
    print(i,'\t',idx,'\t',len(allAlbums))

    
    for dbName in toMatch:
        mdbMatcher = matchDBArtist(maindb)
        mdbMatcher.setArtistInfo(myArtistName, myArtistID, allAlbums)
        #mdbMatcher.findPotentialArtistNameMatches()
        mc = mdbMatcher.findPotentialArtistAlbumMatchesByDB(dbName)
        if mc.matchID is not None:
            if results.get(idx) is None:
                results[idx] = {}
            results[idx][dbName] = mc.matchID
            print('\t\t',i,'/',nDF,'\t',idx,' --> \t',dbName,'\t',mc.matchID)
            saves += 1

    if saves > 25:
        break

    continue
    
    
    # New DB
    if row[newDB] is None:
        allMusicDataNewDBID    = allMusicData.getDict()[newDB]
        if allMusicDataNewDBID is not None:
            newDBAlbums = maindb.dbdatamap[newDB].getArtistAlbums(allMusicDataNewDBID, flatten=True)
            mdbMatcher.setArtistInfo(allMusicName, allMusicID, newDBAlbums)
            mc = mdbMatcher.findPotentialArtistAlbumMatchesByDB(newDB)
            matchID = mc.matchID
            if matchID is not None:
                print("Adding From AllMusic")
                #mdbmap.addArtistData(myArtistName, myArtistID, newDB, matchID)
                continue
                
                
        musicBrainzDataNewDBID = musicBrainzData.getDict()[newDB]
        if musicBrainzDataNewDBID is not None:
            newDBAlbums = maindb.dbdatamap[newDB].getArtistAlbums(musicBrainzDataNewDBID, flatten=True)
            mdbMatcher.setArtistInfo(musicBrainzName, musicBrainzID, newDBAlbums)
            mc = mdbMatcher.findPotentialArtistAlbumMatchesByDB(newDB)
            matchID = mc.matchID
            if matchID is not None:
                print("Adding From MusicBrainz")
                #mdbmap.addArtistData(myArtistName, myArtistID, newDB, matchID)
                continue
    
        print("Not Adding Anything")
        

for idx,artistData in results.items():
    for dbName,dbID in artistData.items():
        dbArtistName = maindb.getArtistDBNameFromID(dbName, dbID)
        print("# {0} \t {1} \t {2}".format(idx[0],dbName,dbArtistName))
        print("#mdbmap.addArtistData(\"{0}\", \"{1}\", \"{2}\", \"{3}\")".format(idx[0], idx[1], dbName, dbID))
        print("")
# <NAME> 	 RockCorner 	 <NAME>
mdbmap.addArtistData("<NAME>", "7b8e61e2831286355c0508034f43a37d", "RockCorner", "23489664")

# 2Gether 	 LastFM 	 2gether
mdbmap.addArtistData("2Gether", "067aaadafbb423759fdc633485ece022", "LastFM", "73759435417")

# Art Pepper 	 Discogs 	 Art Pepper
mdbmap.addArtistData("Art Pepper", "d69fc754cf9c6db1f87d0d39bddb44d3", "Discogs", "251873")

# <NAME> 	 RockCorner 	 <NAME>
mdbmap.addArtistData("<NAME>", "f6af21f6516408272e2c69f23fa6d7bc", "RockCorner", "98950656")

# <NAME> 	 LastFM 	 <NAME>
mdbmap.addArtistData("<NAME>", "91d037a256f8fcca352e7ba015ed7dde", "LastFM", "60632062567")

# The Choir of Trinity College Cambridge 	 Discogs 	 The Choir of Trinity College Cambridge
mdbmap.addArtistData("The Choir of Trinity College Cambridge", "24b3c0e696d44b82ff3dcf3d433aca08", "Discogs", "5230060")

# The Choir of King's College Cambridge 	 Discogs 	 The Choir of King's College Cambridge
mdbmap.addArtistData("The Choir of King's College Cambridge", "3559bca7c05b51850a9011dc0c0cccff", "Discogs", "7350966")

# <NAME> 	 MusicBrainz 	 <NAME>
mdbmap.addArtistData("<NAME>", "5423255356443e5212414eced87a51cf", "MusicBrainz", "253447346849823375595137829318300747372")

# Dorrough 	 Discogs 	 Dorrough
mdbmap.addArtistData("Dorrough", "b0a440e9a57506d1ec9b9feb3c73e409", "Discogs", "1615398")

# Dorrough 	 AllMusic 	 Dorrough
mdbmap.addArtistData("Dorrough", "b0a440e9a57506d1ec9b9feb3c73e409", "AllMusic", "0002132219")

# Dorrough 	 MusicBrainz 	 Dorrough
mdbmap.addArtistData("Dorrough", "b0a440e9a57506d1ec9b9feb3c73e409", "MusicBrainz", "98108632021145924000099772630629089079")
mdbmap.save()
mdbmap.getArtistData('The Highwomen', '66cc9fd99dba1c7879b18721cbfd11e5').show()
mdbmap.addArtistData('The Highwomen', '66cc9fd99dba1c7879b18721cbfd11e5', "AllMusic", "0003859078")
mdbmap.save()

    primaryDBName  = primaryInfo[0]
    primaryDBShort = primaryInfo[1]
    primaryDBArtistName = "{0}ArtistName".format(primaryDBShort)
    primaryDBKey        = "{0}Key".format(primaryDBShort)

    secondaryDBName  = secondaryInfo[0]
    secondaryDBShort = secondaryInfo[1]



    amrcdf = amdf[~amdf[secondaryDBName].isna()][[primaryDBArtistName, primaryDBKey, secondaryDBName]]
    for i,row in amrcdf.iterrows():

        ## Primary DB Key --> Secondary ID
        amDBName = row[primaryDBArtistName]
        amDBKey  = row[primaryDBKey]
        rcDBKey  = row[secondaryDBName]

        if amDBKey is None:
            continue

        #if debug is True:
        #    print(amDBName,'\t',amDBKey,'\t',rcDBKey,'\t-->\t',end="")

        ## Secondary ID --> Secondary DB Key
        primKey = toMatchKeys.get(rcDBKey)
        #if debug is True:
        #    print(primKey)
        #primKey  = mdbmaps[secondaryDBName].getPrimaryKeyFromID(rcDBKey)

        if primKey is None:
            continue
        rcDBName = primKey[0]
        rcDBKey  = primKey[1]



        s = SequenceMatcher(None, str(amDBName), str(rcDBName))

        if debug is True:
            print("{0: <30}{1: <30}{2: <30}{3}".format("", primaryDBName, secondaryDBName, "Match"))
            print("{0: <30}{1: <30}{2: <30}{3}".format("{0} ID   --> {1} ID".format(primaryDBShort, secondaryDBShort), amDBKey, rcDBKey, round(s.ratio(),2)))
            print("{0: <30}{1: <30}{2: <30}".format("{0} Name --> {1} Name".format(primaryDBShort, secondaryDBShort), amDBName, rcDBName))

        continue
for primaryKey,mc in matchResults.items():
    if mc.matchID is not None:
        #mdbmap.addArtistData(primaryKey[0], primaryKey[1], "MusicStack", mc.matchID)
        print(primaryKey,mc.matchID)
        
mdbmap.save()
mydf[mydf.AllMusic.isna() & mydf.MusicBrainz.isna() & mydf.RateYourMusic.isna()]
artistName, artistID = "Cee-Lo", "eac66542fa4809545e96bb7ee94968d3"
artistName, artistID = "N.I.N.A.", "27fe6b13114cb9261849df9995461c8a"
artistName, artistID = "Pentatonix", "47dc3919caaa0f4461aaa942acf78de7"
artistName, artistID = "<NAME>", "a9623107ea03b8bf2013e4d5f5658948"
artistName, artistID = "Hozier", "09c6e3086169271287f71e8531bd6df9"
artistName, artistID = "<NAME>", "ff51ada376d6ddc54bfae2f3b29736cd"
artistName, artistID = "Disturbed", "1fde133fd37227be92ce439ecdb07af1"
artistName, artistID = "MGMT", "66f90deb950bfbbb5bf08eda7317c3ef"
artistName, artistID = "nf", "78d9238c1a217c8bbe8f6c26172fb12d"

if False:
    artistName, artistID = "98 Degrees", "3235abaaa022c42ae63c3fc2078a780c"
    artistName, artistID = "Gwar", "17bcf0bc2768911a378a55f42acedba7"
    artistName, artistID = "P!NK", "068b32bba342b3807e2066c59af25c3e"
    artistName, artistID = "<NAME>", "860a3f382b3c3de4cc3af7cf3ee81ae8"
    artistName, artistID = "Hanson", "1ebd335e8403770be353809cf98cb94e"
    artistName, artistID = "Disclosure", "cffcb1a185b2d7d5c44e9dc785b6bb25"
    artistName, artistID = "t.A.T.u.", "171d19f20e6bfa62ae68fbf359960376"
    artistName, artistID = "<NAME>", "0cd8333d73e58ab2f084e8e66d2e33e8"
    artistName, artistID = "<NAME>", "dee41b7de2e6d53bad6be108bcde1329"
    artistName, artistID = "The Wallflowers", "ad11802450afbe6b4820f1e89e04fac3"
    artistName, artistID = "<NAME>", "3b18c9fc01d81aae93c46a98f4de2265"
    artistName, artistID = "Sugarland", "88018b86c2d69218b08a6bd71a35f398"
    artistName, artistID = "The Jackson 5", "cd70d542131618ca8b44850ac2ad7e18"
mdbMatcher.setArtistInfo(artistName, artistID, None)
mdbMatcher.setThresholds(num=2000, cutoff=0.01)
mdbMatcher.findPotentialArtistNameMatchesByDB("RateYourMusic")
maindb.dbdata["RateYourMusic"]["Disc"].getArtistIDToNameData()
mdbmap.removeArtist("<NAME> & <NAME>", "d2a946454598350c5e0314f301217fef")
mdbmap.save()
artistName, artistID = "nf", "78d9238c1a217c8bbe8f6c26172fb12d"
artistName, artistID = "ee", "08a4415e9d594ff960030b921d42b91e"
artistName, artistID = "<NAME> & <NAME>", "c4b6d5f2427c9f5b0609a85b9eed0e07"
artistName, artistID = "PARTYNEXTDOOR", "79c94267474719dc48e7c213db125c0f"
artistName, artistID = "<NAME>", "d3e21337a5cb70bfe89bcdbe9392b3b2"
artistName, artistID = "Southern Style Dj's", "4dd95f968c47645f65b3dcf190f9ec02"
artistName, artistID = "<NAME>", "53255e92b07c23a68876b102ae93ee2b"
artistName, artistID = "<NAME>", "6d828754ce04480f75d3f5478cb0c06a"
artistName, artistID = "<NAME>", "05404a9be9c7b703a0958ca4b320751a"
artistName, artistID = "Intro To India", "7acc9ed8b383e8d835083f88eb46e012"
artistName, artistID = "<NAME>", "480b64d3917b3ca435089ddd2777d594"
mdbmap.addArtistData(artistName, artistID, "RateYourMusic", '896834')
mdbmap.save()
#mdb.add("I Am Kloot", "AllMusic", "0000256067")
if False:
    ignores  = ["<NAME>", "<NAME>", "<NAME>", "<NAME> & <NAME>", "<NAME>"]
    ignores += ["<NAME>", "<NAME>", "<NAME>", "<NAME>", "<NAME>"]
    ignores += ["<NAME>", "Blyss"]   
    for artistName in ignores:
        mdb.rmArtistDBKey(artistName, "AllMusic")
    mdb.save()
mdb.getArtistDBData('"Weird Al" Yankovic', "LastFM")['ID']
#searchForMutualDBEntries
%load_ext autoreload
%autoreload
if True:
    #mdb = myMusicDBMap(debug=False)
    #mdb.getFullDBData()
    mmm = matchMyMusic(mdb)
    mmm.searchForMutualDBEntriesByDB("RateYourMusic", cutoff=0.90, maxAdds=100, start=-1, modVal=250, maxAlbumsForSearch=500)
#mmm.getArtistStatus()
#mmm.setMusicBase(mmb)
#unknownArtists = mmm.getUnknownArtists()

_, _ = clock("Last Run")

mdb.save()
name = "<NAME>"
dutils = dbdata["LastFM"]["Utils"]
artistID = dutils.getArtistID(name)
print(name,' \t--> ',artistID)
mdb.add(name, "LastFM", artistID)
mdb.save()<jupyter_output><empty_output><jupyter_text>Try Matching Unmatched Albums<jupyter_code>dR = 0.1
rC = 0.99

#for db in ['Discogs', 'AllMusic', 'MusicBrainz', 'AceBootlegs', 'RateYourMusic', 'LastFM', 'DatPiff', 'RockCorner', 'CDandLP', 'MusicStack', 'MetalStorm']:
#for db in ['Discogs', 'MusicBrainz', 'AllMusic', 'LastFM']:
for db in ['LastFM']:
    for albumType in [1]:
        print("="*140)
        mmm.matchMyMusicAlbums(db=db, albumType=albumType, ratioCut=rC, maxCut=rC+dR)
        print("\n\n")
mmm.mmn.moveMyMatchedMusicAlbums(show=True)
mmm.mmn.moveMyMatchedMusicAlbums(show=False)
mmb, mmm, unknownArtists = getMusicStatus(mdb)<jupyter_output><empty_output><jupyter_text>******  Match Albums From My Matched Artists<jupyter_code>dR = 0.1
rC = 0.99
mmm.matchMyMusicAlbums(db="LastFM", albumType=1, ratioCut=rC, maxCut=rC+dR)
mmm.moveMyMatchedMusicAlbums(show=False)
from timeUtils import clock, elapsed
from listUtils import getFlatList
from musicBase import myMusicBase
from matchAlbums import matchAlbums
from ioUtils import getFile, saveFile
from fsUtils import isDir, setDir, mkDir, moveDir
from matchMusicName import myMusicName


class matchMyMusic:
    def __init__(self, mdb, debug=False):
        self.debug = debug
        self.mdb   = mdb
        self.mmb   = myMusicBase()
        self.mmn   = myMusicName()
        
        self.unknownArtists = {}
        self.artistAlbums   = {}
        
        self.matchedAlbums  = {}
        
        
    def setMusicBase(self, mmb):
        self.mmb = mmb
        
        
    def getAlbumStatus(self, force=False):
        self.artistAlbums = self.mmb.getArtistAlbums(force=force)
        

    def getArtistStatus(self):
        start, cmt = clock("Matching All Music Artists")

        ######################################################################
        #### Loop Over My Artists and Paths
        ######################################################################
        for primeDir in self.mmb.getPrimeDirectories():
            for artistName, artistPrimeDirs in self.mmb.getArtistPrimeDirMap(primeDir).items():
                if self.debug:
                    print("{0: <50}{1}".format(artistName,artistPrimeDirs))


                ######################################################################
                #### Get Database IDs
                ######################################################################
                isKnown = self.mdb.isKnown(artistName)
                if isKnown is False:
                    self.unknownArtists[artistName] = artistPrimeDirs
                    if self.debug:
                        print("\tUnknown (All)     --> {0}".format(artistName))
                        

        elapsed(start, cmt)
        print("Found {0} unknown artists".format(len(self.unknownArtists)))
        print("Found {0} total artists".format(len(self.artistAlbums)))

        
        
    def getUnknownArtists(self):
        return self.unknownArtists
    
    
    
    def getArtistNameMatchedDirs(self):
        self.artistMatchedDirs = {}
        for primeDir in self.mmb.getPrimeDirectories():
            self.artistMatchedDirs.update(self.mmb.getArtistPrimeDirMap(primeDir))
            

    
    def matchMyMusicAlbumsByArtist(self, db, artistName, albumType=None, ratioCut=0.95, maxCut=0.1):

        matchedAlbums = {}
        

        ######################################################################
        #### Get Artist Album Data
        ######################################################################
        artistAlbumsData = self.mmb.getArtistAlbumsByArtist(artistName)
    
    
        ######################################################################
        #### Get Unmatched Albums
        ######################################################################
        unMatchedAlbums = self.mmb.getUnMatchedAlbumsByArtist(artistName)
        dirval = self.mmb.getArtistMusicDir(artistName)
        if len(unMatchedAlbums) == 0:
            return matchedAlbums
            
            
        ######################################################################
        #### Loop Over Artist Name <-> Prime Map Items
        ######################################################################
        if self.mdb.isKnown(artistName) is True:
            myMusicData = self.mdb.getArtistData(artistName)
            try:
                artistID = myMusicData[db]["ID"]
            except:
                return matchedAlbums
        else:
            return matchedAlbums
            


        ######################################################################
        #### Get Database Albums
        ######################################################################
        artistDBAlbumsFromID = self.mdb.getArtistAlbumsFromID(db, artistID)

        
        ######################################################################
        #### Loop over my albums
        ######################################################################
        for myAlbumName in unMatchedAlbums:

            bestMatchVal = {"Ratio": ratioCut, "Dir": None, "Album": None}

            for mediaType, mediaTypeAlbums in artistDBAlbumsFromID.items():
                if albumType is not None:
                    if mediaType not in self.mdb.getDBAlbumTypeNames(db, albumType):
                        continue

                if self.debug:
                    print("\tMy album: {0}".format(myAlbumName))
                myFormattedAlbum = self.mmn.formatAlbum(myAlbumName, mediaType)

                ma = matchAlbums(cutoff=ratioCut)
                ma.match([myFormattedAlbum], mediaTypeAlbums)

                if ma.maxval < ratioCut or ma.maxval > ratioCut+maxCut:
                    continue
                if ma.maxval < bestMatchVal["Ratio"]:
                    continue

                bestMatch = ma.getBestMatch(myFormattedAlbum)

                bestMatchVal = {"Ratio": ma.maxval, "Dir": dirval, 
                                "Album": {"Name": bestMatch["Name"],
                                          "Code": bestMatch["Code"],
                                          "MediaType": mediaType}}
                matchedAlbums[myAlbumName] = bestMatchVal
                #print("{0: <30}{1: <15}{2: <30} --> {3}".format(artistName, db, myAlbumName, bestMatchVal["Album"]))
                #bestMatchVal["Match"].show(debug=True)
                    
        return matchedAlbums

                
    
    def matchMyMusicAlbums(self, db, albumType=1, ratioCut=0.95, maxCut=0.1):
        self.matchedAlbums = {}

        start, cmt = clock("Checking for Albums Matches Against {0} DB".format(db))
        
        
        print("{0: <40}{1: <15}{2: <45} --> {3}".format("Artist", "Database", "Album Name", "Matched Album"))

        ######################################################################
        #### Get Map of Artists and Unmatched Albums
        ######################################################################
        artistNames = self.mmb.getArtists()
        #artistAlbums = self.mmb.getArtistAlbums()


        ######################################################################
        #### Loop Over Artist Name <-> Prime Map Items
        ######################################################################
        for artistName in artistNames:
            matchedAlbums = self.matchMyMusicAlbumsByArtist(db, artistName, albumType, ratioCut, maxCut)
            if len(matchedAlbums) > 0:
                if self.matchedAlbums.get(db) is None:
                    self.matchedAlbums[db] = {}
                self.matchedAlbums[db][artistName] = matchedAlbums
                for myAlbumName,bestMatchVal in matchedAlbums.items():
                    print("{0: <40}{1: <15}{2: <45} --> {3}".format(artistName, db, myAlbumName, bestMatchVal["Album"]))

            
        elapsed(start, cmt)

        saveFile(ifile=self.mmn.moveFilename, idata=self.matchedAlbums, debug=True)
        print("Found {0} music <-> discogs albums maps".format(len(self.matchedAlbums)))
        

    def matchUnknownArtists(self, albumType=1, ratioCut=0.95):
        unknownArtists = self.getUnknownArtists()
        for unknownArtist in unknownArtists.keys():
            print("# ===>",unknownArtist)
            retval = self.matchUnknownArtist(unknownArtist, albumType, ratioCut)

            for db,dbdata in retval.items():
                bestMatch = {"ID": None, "Matches": 0, "Score": 0.0}
                for artistDBID,artistDBData in dbdata.items():
                    for mediaType,ma in artistDBData.items():
                        if ma.near == 0:
                            continue
                        if ma.near > bestMatch["Matches"]:
                            bestMatch = {"ID": artistDBID, "Matches": ma.near, "Score": ma.score}
                        elif ma.near == bestMatch["Matches"]:
                            if ma.score > bestMatch["Score"]:
                                bestMatch = {"ID": artistDBID, "Matches": ma.near, "Score": ma.score}

                if bestMatch["ID"] is not None:
                    print("mdb.add(\"{0}\", \"{1}\", \"{2}\")".format(unknownArtist, db, bestMatch["ID"]))
            
            
    def matchUnknownArtist(self, unknownArtist, albumType=None, ratioCut=0.95):
        ######################################################################
        #### Get Unknown Artist Albums and Potential DB Artists
        ######################################################################
        unMatchedAlbums = self.mmb.getUnMatchedAlbumsByArtist(unknownArtist)
        artistNameDBIDs = self.mdb.getArtistIDs(unknownArtist)
        
        #print(unknownArtist)
        #print(unMatchedAlbums)
        #print(artistNameDBIDs)
        #return

        
        ######################################################################
        #### Get Database Albums
        ######################################################################
        matches = {}
        for db,artistDBartists in artistNameDBIDs.items():
            
            dbMatches = {}
            for artistDBartist,artistDBIDs in artistDBartists.items():
                for artistDBID in artistDBIDs:
                    dbMatches[artistDBID] = {}
                    artistDBAlbumsFromID = self.mdb.getArtistAlbumsFromID(db, artistDBID)

                    for mediaType, mediaTypeAlbums in artistDBAlbumsFromID.items():
                        if mediaType not in self.mdb.getDBAlbumTypeNames(db, albumType):
                            continue

                        ma = matchAlbums(cutoff=ratioCut)
                        ma.match(unMatchedAlbums, mediaTypeAlbums)
                        #ma.show(debug=True)
                        
                        dbMatches[artistDBID][mediaType] = ma
                        
            matches[db] = dbMatches
            
        return matches
    
    def manuallyMatchUnknownArtist(self, unknownArtist, cutoff=0.8):
        ######################################################################
        #### Get Unknown Artist Albums and Potential DB Artists
        ######################################################################
        unMatchedAlbums = self.mmb.getUnMatchedAlbumsByArtist(unknownArtist)
        artistNameDBIDs = self.mdb.getArtistIDs(unknownArtist, cutoff=cutoff)
        
        print("Unknown Artist:   {0}".format(unknownArtist))
        try:
            print("UnMatched Albums: {0}".format(", ".join(unMatchedAlbums)))
        except:
            print("Could not show the unMatched Albums below:")
            print("-> ",unMatchedAlbums," <-")
        print("="*50)
        print(artistNameDBIDs)
        for db,artistDBartists in artistNameDBIDs.items():
            print("="*50)
            print("   {0}".format(db))
            for artistDBartist,artistDBIDs in artistDBartists.items():
                print("      {0}".format(artistDBartist))
                for artistDBID in artistDBIDs:
                    artistDBAlbumsFromID = self.mdb.getArtistAlbumsFromID(db, artistDBID)
                    albums = [list(mediaTypeAlbums.values()) for mediaTypeAlbums in artistDBAlbumsFromID.values()]
                    print("mdb.add(\"{0}\", \"{1}\", \"{2}\")".format(unknownArtist, db, artistDBID))
                    print("         {0: <45}\t{1}".format(artistDBID, getFlatList(albums)))
                    

                    
    def getArtistDBMatchLists(self, dbartist):
        dbArtistData   = self.mdb.getArtistData(dbartist)
        retval = {"Matched": [], "Unmatched": []}
        albumTypesData = {k: [] for k in [1,2,3,4]}
        for db,dbIDdata in dbArtistData.items():
            try:
                dbID = dbIDdata["ID"]
                retval["Matched"].append(db)
            except:
                retval["Unmatched"].append(db)
        return retval
    
                    
    def getMatchedArtistAlbumsFromDB(self, dbartist, merge=True):
        dbArtistData   = self.mdb.getArtistData(dbartist)
        dbsToSearch    = self.getArtistDBMatchLists(dbartist)
        albumTypesData = {k: [] for k in [1,2,3,4]}
        for db in dbsToSearch["Matched"]:
            dbIDdata = dbArtistData[db]
            try:
                dbID = dbIDdata["ID"]
            except:
                raise ValueError("This db {0} should already be known for {1}".format(db, dbartist))

            dbAlbumsData = self.mdb.getArtistAlbumsFromID(db, dbID)

            for albumType in albumTypesData.keys():
                for mediaType, mediaTypeAlbums in dbAlbumsData.items():
                    if mediaType not in self.mdb.getDBAlbumTypeNames(db, albumType):
                        continue                
                    #print(db,albumType,mediaType,mediaTypeAlbums)
                    albumTypesData[albumType] += list(mediaTypeAlbums.values())

        albumTypesData = {k: list(set(v)) for k,v in albumTypesData.items()}

        ############################
        ## Merge Albums
        ############################
        if merge is True:
            artistAlbums = getFlatList(albumTypesData.values())
        else:
            artistAlbums = albumTypesData

        return artistAlbums

            
    def searchForMutualDBEntries(self, cutoff=0.8, maxAdds=50, start=None):
        ######################################################################
        #### Get Map of Artists and Unmatched Albums
        ######################################################################
        dbartists = self.mdb.getArtists()
        cnts      = 0
        print("Searching for mutual DB matches for {0} artists".format(len(dbartists)))
        for ia,dbartist in enumerate(dbartists):
            if start is not None:
                if ia < start:
                    continue
            if ia % 100 == 0:
                print("## {0}/{1}".format(ia,len(dbartists)))
            if cnts >= maxAdds:
                break
            artistAlbums = self.getMatchedArtistAlbumsFromDB(dbartist, merge=True)
            dbsToSearch  = self.getArtistDBMatchLists(dbartist)

            usefulDBs          = ['Discogs', 'MusicBrainz', 'AllMusic', 'LastFM']
            usefulDBsToSearch  = list(set(dbsToSearch["Unmatched"]).intersection(set(usefulDBs)))


            ########################################################
            ## Loop Over Unmatched DBs
            ########################################################
            for db in usefulDBsToSearch:
                dbMatches = {}
                artistDBartists = self.mdb.getArtistDBIDs(dbartist, db, num=10, cutoff=cutoff, debug=False)
                
                for artistDBartist,artistDBIDs in artistDBartists.items():
                    #print('  ',db,'\t',artistDBartist)
                    for artistDBID in artistDBIDs:
                        #print('    ',artistDBID)
                        dbMatches[artistDBID] = {}
                        artistDBAlbumsFromID = self.mdb.getArtistAlbumsFromID(db, artistDBID)

                        albumTypesData = {k: [] for k in [1,2,3,4]}
                        for albumType in albumTypesData.keys():
                            for mediaType, mediaTypeAlbums in artistDBAlbumsFromID.items():
                                if mediaType not in self.mdb.getDBAlbumTypeNames(db, albumType):
                                    continue
                                albumTypesData[albumType] += list(mediaTypeAlbums.values())

                        albumTypesData = {k: list(set(v)) for k,v in albumTypesData.items()}
                        dbArtistAlbums = getFlatList(albumTypesData.values())
            

                        ma = matchAlbums(cutoff=cutoff)
                        ma.match(artistAlbums, dbArtistAlbums)
                        #ma.show(debug=True)
                        
                        dbMatches[artistDBID] = ma
                
                if len(dbMatches) > 0:
                    bestMatch = {"ID": None, "Matches": 0, "Score": 0.0}
                    for artistDBID,ma in dbMatches.items():
                        if ma.near == 0:
                            continue
                        if ma.near > bestMatch["Matches"]:
                            bestMatch = {"ID": artistDBID, "Matches": ma.near, "Score": ma.score}
                        elif ma.near == bestMatch["Matches"]:
                            if ma.score > bestMatch["Score"]:
                                bestMatch = {"ID": artistDBID, "Matches": ma.near, "Score": ma.score}

                    if bestMatch["ID"] is not None:
                        cnts += 1                 
                        print("mdb.add(\"{0}\", \"{1}\", \"{2}\")".format(dbartist, db, bestMatch["ID"]))
                        



%load_ext autoreload
%autoreload
mmm = matchMyMusic(mdb)

artistData
artistData     = mdb.getArtistData("<NAME>")
albumTypes     = [1,2,3,4]
albumTypesData = {k: [] for k in albumTypes}
fullAlbumTypesData = {k: {} for k in albumTypes}
for db,dbArtistData in artistData.items():
    try:
        artistDBID = dbArtistData['ID']
    except:
        continue
    artistDBAlbumsFromID = mdb.getArtistAlbumsFromID(db, artistDBID)

    for albumType in albumTypes:
        for mediaType, mediaTypeAlbums in artistDBAlbumsFromID.items():
            if mediaType not in mdb.getDBAlbumTypeNames(db, albumType):
                continue
            albumTypesData[albumType] += list(mediaTypeAlbums.values())
            if fullAlbumTypesData[albumType].get("{0}-{1}".format(db,mediaType)) is None:
                fullAlbumTypesData[albumType]["{0}-{1}".format(db,mediaType)] = list(mediaTypeAlbums.values())

albumTypesData = {k: list(set(v)) for k,v in albumTypesData.items()}
dbArtistAlbums = getFlatList(albumTypesData.values())
studio = {"AllMusic": ["Albums"], "Discogs": ["Albums"], "MusicBrainz": ["Album"], "RateYourMusic": ["Album"], "RockCorner": ["Albums"]}
albumsMap    = {}
fullAlbumMap = Counter()

dbmapping     = {}
invdbmapping  = {}
for db,mediaTypes in studio.items():
    dbmapping[db]    = {}
    invdbmapping[db] = {}

    for mediaType in mediaTypes:
        dbAlbums      = fullAlbumTypesData[1]["{0}-{1}".format(db,mediaType)]
        for k in dbAlbums:
            dbmapping[db][k.upper()] = k
            dbmapping[db][k] = k.upper()
            fullAlbumMap[k.upper()] += 1
        
#fullAlbumMap
core = set()
noncore = set()
nearest = {k: findNearest(k, list(fullAlbumMap.keys()), 2, 0.9) for k in fullAlbumMap.keys()}
for k,v in nearest.items():
    if len(v) == 2:
        if fullAlbumMap[v[0]] > fullAlbumMap[v[1]]:
            core.add(v[0])
        elif fullAlbumMap[v[1]] > fullAlbumMap[v[0]]:
            core.add(v[1])
        else:
            noncore.add(v[0])
            noncore.add(v[1])
            #print(k,v,[fullAlbumMap[v2] for v2 in v])
    else:
        if fullAlbumMap[k] > 1:
            core.add(k)
        else:
            noncore.add(k)
            #print(k,fullAlbumMap[k])
[core
noncore
fullAlbumTypesData[1]
## Basic stuff
%load_ext autoreload
%autoreload

from musicBase import myMusicBase
mmb = myMusicBase()
mmb.findArtistAlbums()<jupyter_output><empty_output><jupyter_text>******<jupyter_code>DatPiffArtists = maindb.getDBData('DatPiff')['Disc'].getMasterSlimArtistDiscogsDB()['DiscArtist']
retval = {artist: mdb.getArtistDBIDs(db="DatPiff", artistName=artist, cutoff=0.9) for artist in mmb.getArtists()}
retval = {artist: v for artist, v in retval.items() if len(v) > 0}
saveFile(idata=retval, ifile="datPiffMatch.yaml")
matches = getFile("datPiffMatch.yaml")
for artist, match in matches.items():
    values = list(match.values())
    if values[0] is None:
        print(artist)
    if len(values[0]) != 1:
        print(artist)
    print(values[0][0])
    mdb.add(artist, "DatPiff", str(values[0][0]))
mdb.save()
mmm.matchUnknownArtist('Notorious B.I.G.')<jupyter_output><empty_output><jupyter_text>UnMatch Artist<jupyter_code>def unMatchArtist(artistName):
    from os.path import join
    from fsUtils import removeDir, isFile, setFile, removeFile

    for musicDir in getMatchedDirs():
        dirval        = join(musicDir, getPrimeDirectory(artistName), artistName)
        if not isDir(dirval):
            continue
            
        matchedDir    = setDir(dirval, "Match")
        mediaTypeDirs = findDirs(matchedDir)
        for mediaTypeDir in mediaTypeDirs:
            for matchDir in findDirs(mediaTypeDir):
                albumName = getUnMatchedDirName(getDirBasics(matchDir)[-1], mediaTypeDir)

                srcDir = matchDir
                dstDir = setDir(dirval, albumName)
                if isDir(dstDir):
                    i = 0
                    while not isDir(dstDir):
                        dstDir = "{0} [Fix-{1}]".format(setDir(dirval, albumName), i)
                        i += 1

                moveDir(srcDir, dstDir, debug=True)

            if isDir(mediaTypeDir):
                DS_Store = setFile(mediaTypeDir, ".DS_Store")
                if isFile(DS_Store):
                    removeFile(DS_Store, debug=True)
                removeDir(mediaTypeDir, debug=True)

        if isDir(matchedDir):
            DS_Store = setFile(matchedDir, ".DS_Store")
            if isFile(DS_Store):
                removeFile(DS_Store, debug=True)
            removeDir(matchedDir, debug=True)
            
unMatchArtist("Sweet")
######################################################################
#### Loop Over Prime Directories
######################################################################
for primeDir in getPrimeDirectories():
    artistPrimeDirMap = getArtistPrimeDirMap(primeDir)

    ######################################################################
    #### Loop Over Artist Name <-> Prime Map Items
    ######################################################################
    for artistName, artistPrimeDirs in artistPrimeDirMap.items():
        unMatchArtist(artistName)<jupyter_output><empty_output><jupyter_text>******  Merge DBs After Finding Matches<jupyter_code>retval = searchForMutualDBEntries(mdb, minI=-1, cutoff=0.7, maxR=3000)
retval
if len(retval) > 0:
    for artistName,artistResult in retval.items():
        for db,dbResult in artistResult.items():
            mdb.add(artistName, db, dbResult["ID"])
    mdb.save()
mdb.getArtistData("A-Mafia")
for unknownArtist in unknownArtists.keys():
    print(unknownArtist)
db = "AllMusic"
for unknownArtist in unknownArtists.keys():
    artistNameDBIDs = mdb.getArtistDBIDs(unknownArtist, db, cutoff=0.99)
    if len(artistNameDBIDs) == 1:
        mdb.add(unknownArtist, db, artistNameDBIDs[unknownArtist][0])
        #print(unknownArtist,'\t',artistNameDBIDs)
mdb.save()
artistName = "Sweet"
db = "AllMusic"
artistName = "Sweet"
db = "AllMusic"
unMatchedAlbums = mmb.getArtistAlbumsByArtist(artistName).getUnmatched()
artistNameDBIDs = mdb.getArtistDBIDs(artistName, db)
resultD = {"ID": None, "Matches": 0, "Score": 0.0, "Best": None}

for artistName,artistDBIDs in artistNameDBIDs.items():
    print(artistName,artistDBIDs)
    for dbArtistID in artistDBIDs:
        print('\t',dbArtistID)
        dbArtistIDAlbums = mdb.getArtistAlbumsFromID(db, dbArtistID, flatten=True)
        print("\t\t",dbArtistIDAlbums)

        
        ### Match Albums (if possible)
        ma = matchAlbums()
        ma.match(unMatchedAlbums, dbArtistIDAlbums)
        if debug:
            print("\t\t{0: <45}{1}\t{2}\t{3}\t{4}".format(dbArtistID, len(dbArtistIDAlbums), ma.near, ma.score, ma.maxval))
        if ma.near < resultD["Matches"]:
            continue
        if ma.score < max([resultD["Score"], cutoff]):
            continue
        resultD = {"ID": dbArtistID, "Matches": ma.near, "Score": ma.score, "Best": ma}
        if debug:
            print("\t\t{0: <45}{1}\t{2}\t{3}\t{4} <-- Match".format(dbArtistID, len(artistAlbums), ma.near, ma.score, ma.maxval))




from matchAlbums import matchAlbums

def searchForArtistDBEntries(mdb, artistName):
    mmb.getArtistAlbumsByArtist("Sweet").getUnmatched()
    albums = 


def searchForArtistAlbumsDBEntries(mdb, artistName, albums, dbsToMatch=None, cutoff=0.7, num=10, debug=False):
    retval     = {}
    

    ######################################################################
    #### Set Known Albums
    ######################################################################
    knownArtistAlbums = albums
    
    
    ######################################################################
    #### Loop Over Missing DBs
    ######################################################################
    if dbsToMatch is None:
        dbsToMatch = mdb.getDBs()
    for db in dbsToMatch:        
        artistDBIDs = mdb.getArtistDBIDs(artistName, db, cutoff=cutoff, num=num, debug=num)
        print("{0: <20}".format(db), end="\t")
        if "Full" in debug:
            print("Found {0} possible artists in DB".format(len(artistDBIDs)))
        else:
            print("")
        
        
        ######################################################################
        #### Search For Matches in Possible IDs
        ######################################################################
        resultD = {"ID": None, "Matches": num, "Score": 0.0, "Best": None}
        for dbArtistName, dbArtistIDs in artistDBIDs.items():
            for dbArtistID in dbArtistIDs:
                dbArtistIDAlbums = mdb.getArtistAlbumsFromID(db, dbArtistID, flatten=True)
                                
                ma = matchAlbums()
                ma.match(knownArtistAlbums, dbArtistIDAlbums)
                if "ID" in debug or "Full" in debug:
                    print("\t\t{0: <45}{1}\t{2}\t{3}\t{4}".format(dbArtistID, len(dbArtistIDAlbums), ma.near, ma.score, ma.maxval))
                if ma.near < resultD["Matches"]:
                    continue
                if ma.score < max([resultD["Score"], cutoff]):
                    continue
                resultD = {"ID": dbArtistID, "Matches": ma.near, "Score": ma.score, "Best": ma}
                print("\t\t{0: <45}{1}\t{2}\t{3}\t{4} <-- Match".format(dbArtistID, len(artistAlbums), ma.near, ma.score, ma.maxval))

                
        if resultD["ID"] is not None:
            print("\t\t{0: <45}{1}\t{2} <====================================== Best Match".format(resultD["ID"], resultD["Matches"], resultD["Score"]))
            retval[db] = {'ID': resultD["ID"], 'Name': None}
            if "Full" in debug:
                print("\t\t =====>",retval[db])
        else:
            if "Full" in debug:
                print("\t\t =====> No Match")
            retval[db] = None
            
    return retval



def searchForMutualArtistDBEntries(mdb, artistName, num=2, cutoff=0.8, debug=[None]):
    retval     = {}

    
    ######################################################################
    #### Determine Albums To Match
    ######################################################################
    artistAlbums = []
    dbsToMatch   = []
    dbMatches    = mdb.getArtistDataIDs(artistName)
    knownDBs     = []
    for db,artistID in dbMatches.items():
        if artistID is not None:
            artistAlbums.append(mdb.getArtistAlbumsFromID(db, artistID, flatten=True))
            knownDBs.append(db)
        else:
            dbsToMatch.append(db)
    from listUtils import getFlatList
    knownArtistAlbums   = list(set(getFlatList(artistAlbums)))
    print("Searching for matches:  [{0}] using [{1}] albums collected from [{2}] dbs".format(artistName, len(knownArtistAlbums), len(artistAlbums)))
    print("  Will search for matches in these DBs: {0}".format(dbsToMatch))
    
    
    ######################################################################
    #### Loop Over Missing DBs
    ######################################################################
    retval = searchForArtistAlbumsDBEntries(mdb, artistName, knownArtistAlbums, dbsToMatch, cutoff, num, debug)
    return retval
            
            
def searchForMutualDBEntries(mdb, num=2, cutoff=0.8, debug=[None], minI=-1, maxR=50):
    retval = {}
    nR = 0
    
    musicArtists = mdb.getArtists()
    for i, artistName in enumerate(musicArtists):
        if i <= minI:
            continue
        result = searchForMutualArtistDBEntries(mdb, artistName, num, cutoff, debug)
        for db,dbval in result.items():
            if dbval is not None:
                if retval.get(artistName) is None:
                    retval[artistName] = {}
                retval[artistName][db] = dbval
                nR += 1
                
        if nR > maxR:
            break
                
    print("Found {0} new artist matches after looping over {1} artists".format(len(retval), i))
    return retval
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
import numpy as np
from random import random

def r(size=0.1):
    val = size*(random() - 0.5)
    return val

def func(x, a, b):
    return (a + b*np.sin(np.pi*x/24.0))
days = 7
x = np.linspace(0,24*days,250*days)
y = [func(t, a=1.0, b=2.0) for t in x]
ygen = y + 0.2 * np.random.normal(size=x.size)
xdata = x
ydata = ygen
popt, pcov = curve_fit(func, xdata, ydata)
popt
plt.plot(x,y, 'r--', label='truth')
plt.plot(x,ygen, 'g--', label='data')
plt.xlabel('Hour of Week')
plt.ylabel('Energy Usage [arbitrary units]')
plt.plot(xdata, func(xdata, *popt), 'b-', label='fit: a=%5.2f, b=%5.2f' % tuple(popt))
plt.legend()
plt.show()
predicted = [4, 25,  0.75, 11]
observed  = [3, 21, -1.25, 13]
if len(predicted) != len(observed):
    raise ValueError("Must be equal lengths")
residual = list(zip(predicted, observed))
N = len(residual)
import numpy as np
np.sqrt(sum([(item[0] - item[1])**2 for item in list(residual)])/N)
import numpy as np
import pandas as pd


# you can use this table as an example
distr_table = pd.DataFrame({
    'X': [0, 0, 1, 1],
    'Y': [1, 2, 1, 2],
    'pr': [0.25, 0.25, 0.15, 0.35]
})

class CheckIndependence:

    def __init__(self):
        self.version = 1

    def check_independence(self, distr_table: pd.DataFrame):
        # write your solution here
        ## Initialize output
        output = {"are_independent": None, "cov": None, "corr": None}
        
        ### Mean X and Sigma X
        muX  = 0
        for Xval,df in distr_table.groupby('X'):
            prX   = df['pr'].sum()
            muX  += Xval*prX
        sigX = 0
        for Xval,df in distr_table.groupby('X'):
            prX   = df['pr'].sum()
            sigX += prX * (Xval - muX)**2
        sigX = np.sqrt(sigX)

        ### Mean Y and Sigma Y
        muY  = 0
        for Yval,df in distr_table.groupby('Y'):
            prY  = df['pr'].sum()
            muY += Yval*prY
        sigY = 0
        for Yval,df in distr_table.groupby('Y'):
            prY  = df['pr'].sum()
            sigY += prY * (Yval - muY)**2
        sigY = np.sqrt(sigY)

        print(sigX)
        print(sigY)
            
        ### Covariance Calculation
        cov = 0
        for i,row in distr_table.iterrows():
            pr   = row.pr
            xval = row.X
            yval = row.Y
            cov += pr*(xval - muX)*(yval - muY)            
            
        ### Correlation Calculation
        corr = cov / (sigX * sigY)
        
        
        
        ### Independence Calculation
        diff = 0
        for i,row in distr_table.iterrows():
            pr   = row.pr
            xval = row.X
            yval = row.Y

            prX = distr_table[distr_table['X'] == xval]['pr'].sum()
            prY = distr_table[distr_table['Y'] == yval]['pr'].sum()

            prXY = prX*prY
            diff += np.abs(prXY - pr)

        indy = True
        if diff > 1e-4:
            indy = False
        
        
        
        output["cov"]  = cov
        output["corr"] = corr
        output["are_independent"] = indy
        
        return output
ci = CheckIndependence()
ci.check_independence(distr_table)
ci
distr_table['X'].unique()
muX  = 0
for Xval,df in distr_table.groupby('X'):
    prX   = df['pr'].sum()
    muX  += Xval*prX
sigX = 0
for Xval,df in distr_table.groupby('X'):
    sigX += prX * (Xval - muX)**2
sigX = np.sqrt(sigX)

muY  = 0
for Yval,df in distr_table.groupby('Y'):
    prY  = df['pr'].sum()
    muY += Yval*prY
sigY = 0
for Yval,df in distr_table.groupby('Y'):
    prY  = df['pr'].sum()
    sigY += prY * (Yval - muY)**2
sigY = np.sqrt(sigY)
cov = 0
for i,row in distr_table.iterrows():
    pr   = row.pr
    xval = row.X
    yval = row.Y
    cov += pr*(xval - muX)*(yval - muY)

diff
prX = {}
for Xval,df in distr_table.groupby('X'):
    prX[Xval] = df['pr'].sum()
prX
distr_table[distr_table['X'] == 0]['pr'].sum()
def analyzeRenames(dbRenames):
    redos = {}
    dels = []
    for oldername,bestname in dbRenames.items():
        if oldername in manDB.artistNameDB.keys() and bestname not in manDB.artistNameDB.keys():
            print(oldername,bestname)

            redos[bestname] = oldername
            dels.append(oldername)
            print('\t',oldername,'\t',bestname)

    print("# Renames: {0}".format(len(dbRenames)))
    print("# Dels: {0}".format(len(dels)))
    for oldername in dels:
        del dbRenames[oldername]

    print("# Redos: {0}".format(len(redos)))
    dbRenames.update(redos)
    print("# Renames: {0}".format(len(dbRenames)))
    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")




    def isAscii(ele):
        return len(ele) == len(ele.encode())

    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if isAscii(v) and not isAscii(k):
            if '’' in k or "“" in k:
                continue
            redos[v] = k
            dels.append(k)
            print('\t',k,'\t',v)
    print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    print(len(dbRenames))
    dbRenames.update(redos)
    print(len(dbRenames))
    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")




    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if " the " in v and " The " in k:
            if k == v.replace(" the ", " The "):
                redos[v] = k
                dels.append(k)
                print('\t',k,'\t',v)

    print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    print(len(dbRenames))
    dbRenames.update(redos)
    print(len(dbRenames))
    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")




    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if '’' in v or "“" in v:
            if not '’' in k and not "“" in v:
                redos[v] = k
                dels.append(k)
                print('\t',k,'\t',v)

    print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    print(len(dbRenames))
    dbRenames.update(redos)
    print(len(dbRenames))


    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")








    def isAscii(ele):
        return len(ele) == len(ele.encode())

    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if isAscii(v) and not isAscii(k):
            if '’' in k or "“" in k:
                continue
            redos[v] = k
            dels.append(k)
            print('\t',k,'\t',v)
    print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    print(len(dbRenames))
    dbRenames.update(redos)
    print(len(dbRenames))


    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")








    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if " the " in v and " The " in k:
            if k == v.replace(" the ", " The "):
                redos[v] = k
                dels.append(k)
                print('\t',k,'\t',v)

    print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    print(len(dbRenames))
    dbRenames.update(redos)
    print(len(dbRenames))


    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")







    dbRenames = getFile("relDBRenames2.yaml")
    redos = {}
    dels = []
    for k,v in dbRenames.items():    
        if '’' in v or "“" in v:
            if not '’' in k and not "“" in v:
                redos[v] = k
                dels.append(k)
                print('\t',k,'\t',v)

    print(len(dbRenames))
    for k in dels:
        del dbRenames[k]

    print(len(dbRenames))
    dbRenames.update(redos)
    print(len(dbRenames))


    saveFile(idata=dbRenames, ifile="relDBRenames2.yaml")
    print(len(dbRenames))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17276487686459896, 'fraction_numerical': 0.05060369885329159, 'mean_word_length': 3.367336262816489, 'pattern_counts': {'":': 70, '<': 340, '<?xml version=': 0, '>': 308, 'https://': 31, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '356677', 'license': 'MIT', 'n_tokens_mistral': 48343, 'n_tokens_neox': 43968, 'n_words': 7317, 'path': 'Matched.ipynb', 'repo_name': 'tgadf/music', 'size': '126277'}