<jupyter_start><jupyter_text>Initialize<jupyter_code>%load_ext autoreload
%autoreload
discDP = discogs('datpiff')
dutils = datpiffUtils()
artDP  = artistDP()
artsDP = artistsDP(discDP)<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Initialize Multi Artist<jupyter_code>from multiArtist import multiartist
mulArts  = multiartist()<jupyter_output><empty_output><jupyter_text>Download Known Artists<jupyter_code>def getMyMusicDBMap():
    myMusicMap = getFile(ifile="/Users/tgadfort/Documents/code/music/myMusicMap.p")
    print("Found {0} entries".format(len(myMusicMap)))
    return myMusicMap

myMusicMap = getMyMusicDBMap()
%load_ext autoreload
%autoreload
discDP = discogs('datpiff')
dutils = datpiffUtils()
artDP  = artistDP()
artsDP = artistsDP(discDP)
knownFile = "/Volumes/Piggy/Discog/artists-datpiff/known/datPiffKnown.p"
known = getFile(ifile=knownFile)
len(known)<jupyter_output><empty_output><jupyter_text>Individual Search<jupyter_code>artist = "<NAME>"
artsDP.searchRateYouDPusicForArtist(artist)
#known.append(artist)
#saveFile(idata=known, ifile=knownFile)
known = getFile(ifile=knownFile)
len(known)<jupyter_output><empty_output><jupyter_text>Loop Search<jupyter_code>known = getFile(ifile=knownFile)
print("There are {0} searched terms".format(len(known)))
from collections import Counter
from time import sleep
from glob import glob

artistsCntr = Counter()
for ifile in glob("/Volumes/Piggy/Discog/artists-datpiff/*/*.p"):
    if ifile.endswith("datPiffKnown.p"):
        continue
    tmp     = getFile(ifile)
    #print(ifile,'\t',len(tmp))
    results = [x["ArtistName"] for x in tmp]
    for artist in results:
        artists = mulArts.getArtistNames(artist)
        for artist in artists.keys():
            key = artist.title()
            if len(key) > 1 and key not in known:
                artistsCntr[key] += 1
searchTerms = [item[0] for item in artistsCntr.most_common() if item[1] >= 50]
print("There are {0} new searches".format(len(searchTerms)))
for i,searchTerm in enumerate(searchTerms):
    print("\n=====>",i,"/",len(searchTerms),"==>",searchTerm)
    artsDP.searchRateYouDPusicForArtist(searchTerm)
    known.append(searchTerm)
    sleep(3)
print("Saving {0} known search terms".format(len(known)))
saveFile(idata=known, ifile="datPiffKnown.p")
saveFile(idata=known, ifile="/Users/tgadfort/Music/Discog/artists-datpiff/knownSearches.p")<jupyter_output>Saving 1120 known search terms
  --> This file is 10.7kB.
  --> This file is 10.7kB.<jupyter_text>Parse Search Terms<jupyter_code>artsDP.parseArtistFiles(debug=False)<jupyter_output>Found 1122 downloaded search terms
Saving 449 artist IDs to /Volumes/Biggy/Discog/artists-datpiff-db/19-DB.p
  --> This file is 591.2kB.
Saving 449 new artist IDs name data to /Volumes/Biggy/Discog/artists-datpiff-db/metadata/19-Metadata.p
  --> This file is 12.5kB.
Saving 449 new artist IDs media data to /Volumes/Biggy/Discog/artists-datpiff-db/metadata/19-MediaMetadata.p
  --> This file is 85.9kB.
Saving 451 artist IDs to /Volumes/Biggy/Discog/artists-datpiff-db/82-DB.p
  --> This file is 424.9kB.
Saving 451 new artist IDs name data to /Volumes/Biggy/Discog/artists-datpiff-db/metadata/82-Metadata.p
  --> This file is 12.8kB.
Saving 451 new artist IDs media data to /Volumes/Biggy/Discog/artists-datpiff-db/metadata/82-MediaMetadata.p
  --> This file is 73.5kB.
Saving 410 artist IDs to /Volumes/Biggy/Discog/artists-datpiff-db/0-DB.p
  --> This file is 1.2MB.
Saving 410 new artist IDs name data to /Volumes/Biggy/Discog/artists-datpiff-db/metadata/0-Metadata.p
  --> This file is 11.3kB.
S[...]
{'fraction_non_alphanumeric': 0.12774122807017543, 'fraction_numerical': 0.02412280701754386, 'mean_word_length': 4.894991922455573, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '191082', 'license': 'MIT', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1316, 'n_words': 270, 'path': 'notebooks/old/Download Artists DatPiff.ipynb', 'repo_name': 'tgadf/dbmaster', 'size': '3649'}