<jupyter_start><jupyter_text>Initial Tracking List<jupyter_code>def create_artist(row):
    artist = RTArtist(name=row["name"], mbid=row["mbid"], lfm_url=row["url"], artist_rank=int(row["@attr.rank"]))
    if "tracked" in row:
        artist.tracked = row["tracked"]
    return artist

top_artists_df = lfm_get_top_artists_df(user="ianhellen", limit=200)
# list(top_artists_df.apply(create_artist, axis=1).array)[0]

top_artists_df.head()
tracked_select = widgets.SelectMultiple(
    description="Tracked",
    options=top_artists_df["name"].array,
    layout=widgets.Layout(height="300px", width="300px")
)
untrack_btn = widgets.Button(description="Move to Untracked")
track_btn = widgets.Button(description="Move to Tracked")
untracked_select = widgets.Select(
    description="Not Tracked",
    options=[],
    layout=widgets.Layout(height="300px", width="300px")
)

def move_to_untracked(evt):
    del evt
    curr_list = set(untracked_select.options)
    curr_list.update(tracked_select.value)
    untracked_select.options = sorted(curr_list)
    tracked_select.options = [a for a 
                              in tracked_select.options 
                              if a not in untracked_select.options]
def move_to_tracked(evt):
    del evt
    curr_list = set(tracked_select.options)
    curr_list.update(untracked_select.value)
    to_delete_select.options = sorted(curr_list)
    tracked_select.options = [a for a 
                               in untracked_select.options 
                               if a not in tracked_select.options]
    
untrack_btn.on_click(move_to_untracked)
track_btn.on_click(move_to_tracked)
buttons = widgets.VBox([untrack_btn, track_btn])
display(widgets.HBox([tracked_select, buttons, untracked_select]))
top_artists_df["tracked"] = ~ top_artists_df["name"].isin(untracked_select.options)<jupyter_output><empty_output><jupyter_text>Add Tracked artists to DB<jupyter_code>artists_db = RTArtists()
artists_db.add_artists(top_artists_df.apply(create_artist, axis=1).array)
DB_FILE_NAME = "artists_db.pkl"
artists_db = load_db_from_pkl(DB_FILE_NAME)
save_db_to_pkl(artists_db, DB_FILE_NAME)
new_artist = RTArtist(**vars(list(artists_db.artists.values())[1]))
new_artist<jupyter_output><empty_output><jupyter_text>Display<jupyter_code>import ipywidgets as widgets
from IPython.display import Image, display, Markdown
import pdb



def display_artist(selection):
    if (
        selection is None
        or "new" not in selection
        #or not isinstance(selection["new"], int)
    ):
        return
    artist_name = selection["new"]
    
    artist_rec = artists_db[artist_name]
    artist_mbid = artist_rec.mbid
    artist_output.clear_output()
    with artist_output:
        print(artist_name)
        display(Markdown(f"getting data..."))
    
    artist_rec.populate_mb_data()
    new_artists, known_artists = get_new_sim_artists(artists_db, artist_rec)

    artist_output.clear_output()
    fav_check.value = artist_rec.favorite
    art_rating.value = artist_rec.artist_rating.name
    tracked_check.value = artist_rec.tracked
    
    with artist_output:
        display(Markdown(f"## {artist_name}"))
        display(Markdown(f"MusicBrainz mbid: {artist_mbid} URL: {artist_rec.musicbrainz_url}"))
        display(Markdown(f"Discogs URL: {artist_rec.discogs_url}"))
        if artist_rec.img_url:
            display(Image(url=artist_rec.img_url))
        display(Markdown(f"LastFM URL: {artist_rec.lfm_url}"))
        display(Markdown(f"MusicBrainz "))
        
    display_artist_data()

def fetch_artist_data(evt):
    del evt
    artist_rec = get_selected_artist()
    artist_rec.populate_lfm_data()
    artist_rec.populate_mb_data()
    artist_rec.populate_discogs_data()
    if artist_rec.tracked:
        artist_rec.get_similar_artists(artists_db)
    display_artist_data()

def display_artist_data():
    artist_rec = get_selected_artist()
    new_artists, known_artists = get_new_sim_artists(artists_db, artist_rec)
    with artist_output:
        display(Markdown(artist_rec.bio_summary))
        if artist_rec.img_url:
            display(Image(url=artist_rec.img_url))
        display(Markdown(f"LastFM URL: {artist_rec.lfm_url}"))
        
        if artist_rec.aliases:
            display(Markdown("#### Aliases"))
            display(Markdown(", ".join(artist_rec.aliases)))
        if artist_rec.name_variations:
            display(Markdown("#### Name variants"))
            display(Markdown(", ".join(artist_rec.name_variations)))
        if artist_rec.artist_type == ArtistType.Person:
            display(Markdown("#### Member of"))
            display(Markdown(", ".join(artist_rec.member_of.keys())))
        elif artist_rec.artist_type == ArtistType.Group:
            display(Markdown("#### Members"))
            display(Markdown(", ".join(artist_rec.members.keys())))
        display(Markdown("#### Related Artists (new)"))
        display(Markdown(f"**{', '.join(new_artists)}**"))
        display(Markdown("#### Related Artists (known)"))
        display(Markdown(f"*{', '.join(known_artists)}*"))
        
    similar_art_sel.options = new_artists
    visible = "visible" if bool(new_artists) else "hidden"
    similar_art_sel.visibility = visible
    artists_to_add.visibility = visible
    move_to_selected_btn.visibility = visible
    add_artists_btn.visibility = visible

def get_new_sim_artists(artists_db: RTArtists, current_artist):
    known_artists = set(artists_db.artists.keys())
    similar_artists = set(current_artist.similar.keys())
    new_artists = similar_artists - known_artists
    known_artists = known_artists & similar_artists
    return new_artists, known_artists


def move_to_selected(btn_event):
    del btn_event
    cur_selection = list(artists_to_add.options)
    artists_to_add.options=sorted(cur_selection + list(similar_art_sel.value))

def add_new_artists(btn_event):
    del btn_event
    cur_artist = get_selected_artist()
    for new_artist in artists_to_add.options:
        print(artists_to_add.options)
        sim_artist = cur_artist.similar.get(new_artist)
        artist_rec = RTArtist(
            name=new_artist,
            mbid=sim_artist.mbid if sim_artist else None,
            artist_rating = ArtistRating.Like,
            tracked=True)
        artist_rec.populate_lfm_data()
        artist_rec.populate_mb_data()
        artist_rec.populate_discogs_data()
        artist_rec.get_similar_artists(zdb_root.artists_db)
        artists_db.add_artist(artist_rec)
        #print(artist_rec)
    print(sort_opt.value)
    artists_to_add.options=[]

def change_sort_order(opt_event):
    if opt_event["new"] == "Rank":
        artist_list = get_artist_list(artists_db, sort=1)
    else:
        artist_list = get_artist_list(artists_db, sort=1)
    artist_select.options = artist_list

def set_favorite(evt):
    print(evt)
    del evt
    cur_artist = get_selected_artist()
    cur_artist.favorite = fav_check.value
    
def set_rating(evt):
    print(evt)
    del evt
    rating = ArtistRating.parse(art_rating.value)
    cur_artist = get_selected_artist()
    if cur_artist:
        cur_artist.artist_rating = rating

def get_selected_artist():
    if artist_select.label:
        return artists_db[artist_select.label]

def show_hide_tracked(evt):
    del evt
    artist_select.options = get_artist_list(artists_db, get_sort_key())

def get_sort_key():
    return 1 if sort_opt.value == "Rank" else 0

def set_tracked(evt):
    cur_artist = get_selected_artist()
    print(evt)
    if cur_artist:
        cur_artist.tracked = fav_check.value


def save_changes(evt):
    del evt
    save_db_to_pkl(db=artists_db, file_name=DB_FILE_NAME)

def open_close_db():
    global artists_db, db_opened
    return
#     if db_opened:
#         close_zdb()
#         open_close_btn.description = "Open Database"
#     else:
#         artists_db = open_zdb()
#         open_close_btn.description = "Close Database"
#     db_opened = not db_opened

def get_artist_list(artists_db=artists_db, sort=1):
    art_dict = {a.name: a.artist_rank
                for a in artists_db.artists.values()
                if show_tracked_check.value or a.tracked}
    return sorted(art_dict.items(), key=lambda x: x[sort])

# DB/Global Controls
save_btn = widgets.Button(description="Save to Database")
save_btn.on_click(save_changes)
fetch_data_btn = widgets.Button(description="Fetch Artist Data")
fetch_data_btn.on_click(fetch_artist_data)

sort_opt = widgets.RadioButtons(options=["Rank", "Name"], description="Sort by")
sort_opt.observe(change_sort_order, names="value")
show_tracked_check = widgets.Checkbox(description="Show Untracked", value=False)
show_tracked_check.observe(show_hide_tracked, names="value")

# Get Artists
artist_output = widgets.Output(layout={"border": "1px solid black"})
# art_dict = {a.name: a.artist_rank for a in artists_db.artists_db.artists.values()}
# art_dict = {v: k for k,v in top_artists_df["name"].to_dict().items()}

artist_select = widgets.Select(options=get_artist_list(artists_db, sort=1), 
                                  description="Current Artists",
                                  layout=widgets.Layout(width="30%", height="300px"),)

artist_select.observe(display_artist, names="label")


# Artist property controls
fav_check = widgets.Checkbox(description="Favorite", value=False)
fav_check.observe(set_favorite, names="value")
art_rating = widgets.Select(description="Rating", options=list(ArtistRating.__members__.keys()))
art_rating.observe(set_rating, names="value")
tracked_check = widgets.Checkbox(description="Tracked", value=False)
tracked_check.observe(set_tracked, names="value")

# Similar Artist selection
layout_hidden = widgets.Layout(visibility="hidden")
move_to_selected_btn = widgets.Button(description="Add Selection", layout=layout_hidden)
move_to_selected_btn.visible = False
move_to_selected_btn.on_click(move_to_selected)
add_artists_btn = widgets.Button(description="Add Selected to Tracked", layout=layout_hidden)
add_artists_btn.visible = False
add_artists_btn.on_click(add_new_artists)

similar_art_sel = widgets.SelectMultiple(options=[], 
                                         description="Similar Artists",
                                         layout=widgets.Layout(width="300px", height="200px", visibility="hidden"),
                                         style={"description_width": "initial"},
                                         )
artists_to_add = widgets.Select(options=[], 
                                description="Selected",
                                layout=widgets.Layout(width="300px", height="300px", visibility="hidden"),
                                style={"description_width": "initial"},
                                visible = False)

# Display
global_controls = widgets.VBox([fetch_data_btn, sort_opt, show_tracked_check, save_btn])
sim_artists_group = widgets.VBox([similar_art_sel, move_to_selected_btn])
selected_artists_group = widgets.VBox([artists_to_add, add_artists_btn])
artist_props_sub = widgets.VBox([fav_check, art_rating, tracked_check])
artist_props = widgets.HBox([artist_props_sub, sim_artists_group, selected_artists_group])
display(widgets.VBox([widgets.HBox([artist_select, global_controls]),
                      artist_output,
                      artist_props]))
bonobo = artists_db["Bonobo"]
%%debug
bonobo.populate_mb_data()<jupyter_output>NOTE: Enter 'c' at the ipdb>  prompt to continue execution.
> [1;32m<string>[0m(2)[0;36m<module>[1;34m()[0m<jupyter_text>Discogs<jupyter_code>import discogs_client 
do_clnt = discogs_client.Client('IansReleaseTracker/0.1', user_token="<KEY>")
from discogs_client import Artist as DOArtist

def discogs_find_artist(srch_artist, max_checks=0, max_pages=1, case_sensitive=False):
    artists = do_clnt.search(query=srch_artist, type="artist")
    count = 0
    for i in range(0, max_pages):
        if len(artists.page(i)) == 1:
            return artists.page(i)[0]
        for artist in artists.page(i):
            if case_sensitive and artist.name == srch_artist:
                return artist
            elif (not case_sensitive 
                  and artist.name.lower() == srch_artist.lower()):
                return artist
            count += 1
            if max_checks and count > max_checks:
                print(f"{srch_artist} not found")
                return None
            
def discogs_get_artist_by_id(mbid):
    return do_clnt.artist(mbid)

artist = discogs_find_artist("<NAME> (2)")
print(artist)<jupyter_output><Artist 278763 '<NAME> (2)'><jupyter_text>MusicBrainz<jupyter_code>!pip install musicbrainzngs
!pip install python-dateutil
import musicbrainzngs as mbrainz
from dateutil import parser as du_parser
mbrainz.set_useragent(app="IanReleaseTracker", version=0.1, contact="<EMAIL>")

jw_artist_id = "3ae2fb37-8a23-429d-9920-bac811c4fc22"
# result = musicbrainzngs.get_artist_by_id(artist_id,
def mb_get_artist_by_id(artist_id):
    mb_includes=["artist-rels", "release-groups", "url-rels"]
    try:
        result = mbrainz.get_artist_by_id(artist_id, includes=mb_includes)
    except WebServiceError as exc:
        print("Something went wrong with the request: %s" % exc)
    return result["artist"]


def mb_get_discogs_id(mb_artist):
    DICT_KEY = "url-relation-list"
    if DICT_KEY not in mb_artist:
        return []
    discogs_entry = [rel for rel in mb_artist[DICT_KEY] if rel["type"] == "discogs"]
    if discogs_entry:
        return discogs_entry[0]["target"].split("/")[-1]
    return None


def mb_get_group_releases(mb_artist, types=["Album"]):
    DICT_KEY = "release-group-list"
    if DICT_KEY not in mb_artist:
        return []
    for release in mb_artist[DICT_KEY]:
        try:
            release["first-release-date"] = du_parser.parse(release["first-release-date"])
        except (ValueError, OverflowError):
            continue
    if isinstance(types, str) and types.casefold() == "all":
        return mb_artist[DICT_KEY]
    elif isinstance(types, list):
        reqd_types = [req.casefold() for req in types]
        if "all" in reqd_types:
            return mb_artist[DICT_KEY]
        return [rel for rel in mb_artist[DICT_KEY] if rel["primary-type"].casefold() in reqd_types]
    return []


def mb_get_member_of(mb_artist, types=["Album"]):
    DICT_KEY = "artist-relation-list"
    if DICT_KEY not in mb_artist:
        return []
    member_of = []
    for group in [grp for grp in mb_artist[DICT_KEY] if grp["type"] == "member of band"]:
        group_dict = {}
        group_dict.update(group["artist"])
        if "begin" in group:
            group_dict["begin"] = du_parser.parse(group["begin"])
        if "end" in group:
            group_dict["end"] = du_parser.parse(group["end"])
        member_of.append(group_dict)
    return member_of

jwhite_mb = mb_get_artist_by_id('3ae2fb37-8a23-429d-9920-bac811c4fc22')
mb_get_group_releases(jwhite_mb)

# discogs_get_artist_by_id(mb_get_discogs_id(mb_artist))
mb_artist = mb_get_artist_by_id(jw_artist_id)
print(mb_get_discogs_id(mb_artist))

white_stripes = mb_get_artist_by_id("11ae9fbb-f3d7-4a47-936f-4c0a04d3b3b5")

import pandas as pd
pd.DataFrame(mb_get_group_releases(mb_artist, types=["album"]))
[band for band in mb_artist["artist-relation-list"] if band["type"] == "member of band"]
mb_artist
pd.DataFrame(mb_get_member_of(mb_artist))<jupyter_output><empty_output><jupyter_text>Last.FM<jupyter_code>import requests
LFM_KEY_1 = "143c9cc433c0a23bb16ced806e8c40fb"
LFM_KEY = "6ca45596f0b24ec92a77c3281682cd68"
LFM_USER = "ianhellen"

LFM_ROOT = "http://ws.audioscrobbler.com{api}"
LFM_ARTIST = "/2.0/?method=artist.getinfo&artist={artist}&api_key={api_key}{mbid}&format=json"
LFM_TOP_ARTISTS = "/2.0/?method=user.gettopartists&user={user}&api_key={api_key}&limit={limit}&format=json"
LFM_TOP_ARTIST_ALBUMS = "/2.0/?method=artist.gettopalbums&artist={artist}&api_key={api_key}{mbid}&limit={limit}&format=json"
LFM_SIM_ARTIST = "/2.0/?method=artist.getsimilar&artist={artist}&api_key={api_key}&format=json"
import pandas as pd

def lfm_get_top_artists(user: str, api_key: str = LFM_KEY, limit: int = 100):
    last_fm_artists = LFM_ROOT.format(api=LFM_TOP_ARTISTS)
    resp = requests.get(last_fm_artists.format(
        user=user,
        api_key=LFM_KEY,
        limit=limit
    ))
    return resp.json()["topartists"]["artist"]

def lfm_get_top_artists_df(user: str, api_key: str = LFM_KEY, limit: int = 100):
    return pd.json_normalize(lfm_get_top_artists(user, api_key, limit))
    
def lfm_get_artist_info(artist: str,  api_key: str = LFM_KEY, mbid: str = None):
    mbid_opt = f"&mbid={mbid}" if mbid else ""
    url = LFM_ROOT.format(api=LFM_ARTIST)
    resp = requests.get(url.format(
        artist=artist,
        api_key=LFM_KEY,
        mbid=mbid_opt,
    ))
    artist_info = resp.json()
    return artist_info.get("artist", {})

def lfm_artist_similar(artist_info):
    similar_artists = {}
    for sim_artist in artist_info["similar"]["artist"]:
        similar_artists[sim_artist["name"]] = sim_artist
    return similar_artists
#     return artist_info.get("similar", {}).get("artist", {})
    
def lfm_artist_bio(artist_info):
    return artist_info.get("bio", {})
#     return pd.json_normalize(resp.json()["topartists"]["artist"])

def lfm_get_top_artist_albums(artist: str,  api_key: str = LFM_KEY, mbid: str = None, limit: int = 100):
    mbid_opt = f"&mbid={mbid}" if mbid else ""
    url = LFM_ROOT.format(api=LFM_TOP_ARTIST_ALBUMS)
    resp = requests.get(url.format(
        artist=artist,
        api_key=LFM_KEY,
        mbid=mbid_opt,
        limit=limit,
    ))
    album_info = resp.json()
    all_albums = album_info["topalbums"]["album"]
    return [album for album in top_albums if "mbid" in album]


def lfm_get_similar_artists(artist: str, api_key: str = LFM_KEY, num_artists=20):
    url = LFM_ROOT.format(api=LFM_SIM_ARTIST)
    similar_artists = {}
    resp = requests.get(url.format(
        artist=artist,
        api_key=api_key,
        limit=num_artists
    ))
    sim_resp = resp.json()
    if "similarartists" not in sim_resp:
        return {}
    for sim_artist in sim_resp["similarartists"]["artist"][:num_artists]:
        similar_artists[sim_artist["name"]] = sim_artist
    return similar_artists<jupyter_output><empty_output><jupyter_text>Create_artist<jupyter_code>[t.name for t in artists_db2.artists.values() if not t.tracked]
[a for a in init_art_select.options if a not in to_delete_select.options]
top_artists_df.head()<jupyter_output><empty_output><jupyter_text>Add artists to DB<jupyter_code>zdb_root.artists_db.add_artists(top_artists_df.apply(create_artist, axis=1).array)

assert "<NAME>" in zdb_root.artists_db
assert "3ae2fb37-8a23-429d-9920-bac811c4fc22" in zdb_root.artists_db<jupyter_output><empty_output><jupyter_text>Get Releases for Year<jupyter_code>import pandas as pd
nme_url = "https://www.nme.com/releases/albums/{year}-calendar"

release_df = pd.read_html(nme_url.format(year="2019"), header=1)[0]
release_df
datetime.now().year<jupyter_output><empty_output><jupyter_text>Wiki Releases<jupyter_code>def get_wiki_releases
wiki_albums = "https://en.wikipedia.org/wiki/List_of_{year}_albums"

wiki_tbls = pd.read_html(wiki_albums.format(year=year))
wiki_release_df = pd.concat(wiki_tbls[2:-1], ignore_index=True)
len(wiki_tbls)


wiki_release_df["Month"] = wiki_release_df["Release date"].str.extract(r"([^\d]+)")
wiki_release_df["Day"] = wiki_release_df["Release date"].str.extract(r"(\d+)", expand=True)
wiki_release_df["ReleaseDate"] = wiki_release_df.apply(lambda x: pd.to_datetime("-".join([str(x.Day), x.Month, str(year)])), axis=1)
wiki_release_df = wiki_release_df.drop(columns=["Day", "Month", "Release date", "Ref", "Ref."])
wiki_release_df.head()<jupyter_output><empty_output><jupyter_text>NME Releases<jupyter_code>from datetime import datetime
year = "2019"
def fetch_nme_releases(year):
    
    nme_cur_year_url = "https://www.nme.com/releases/albums"
    nme_past_yrs_url = "https://www.nme.com/releases/albums/{year}-calendar"
    if int(year) == datetime.now().year:
        url = nme_cur_year_url
    else:
        url = nme_past_yrs_url
    release_df = pd.read_html(url.format(year=year), header=1)[0]
    rel_date = pd.Timestamp(f"01/01/{year}")
    release_list = []
    for row in release_df.itertuples(index=False, name="Release"):
        if row[0].endswith(str(year)):
            rel_date = pd.Timestamp(row[0])
        elif row[0] == "Artist" and row[1] == "Album":
            continue
        else:
            release_list.append({
                "ReleaseDate": rel_date,
                "Artist": row[0],
                "Album": row[1]
                })
    
    release_df = pd.DataFrame(release_list)

releases2020 = fetch_nme_releases(2020)<jupyter_output><empty_output><jupyter_text>LFM Tests<jupyter_code>rac_info = lfm_get_artist_info(artist="The Raconteurs")
print(rac_info)
rac_similar = lfm_get_similar_artists(artist="The Raconteurs", num_artists=40)

print({k: v["match"] for k, v in rac_similar.items() if float(v["match"]) >= 0.125})

albums = lfm_get_top_artist_albums(artist=rac_info["name"], mbid=rac_info["mbid"])
# top_albums = albums["topalbums"]["album"]
# [(album.get("name"), album.get("playcount"), album.get("mbid")) for album in top_albums if "mbid" in album]
pd.DataFrame(albums)
{k: v for k, v in rac_similar.items() if float(v["match"]) >= 0.25}
top_artists = lfm_get_top_artists(user=LFM_USER, limit=200)
top_artists_df = pd.json_normalize(top_artists)
top_artists_df.head()
zdb_root.artists_db
def create_artist(row):
    artist = RTArtist(name=row["name"], mbid=row["mbid"], lfm_url=row["url"])
    return artist


zdb_root.artists_db.add_artists(top_artists_df.apply(create_artist, axis=1).array)
top_artists_df[top_artist_df["name"] == a["image"][1]["#text"]
"3ae2fb37-8a23-429d-9920-bac811c4fc22"
import ipywidgets as widgets
from IPython.display import Image, display, Markdown
import pdb
def display_artist(selection):
#     with artist_output:
#         print(selection)
    if (
        selection is None
        or "new" not in selection
        #or not isinstance(selection["new"], int)
    ):
        return
    artist_idx = selection["new"]
    
    artist_rec = top_artists_df.iloc[artist_idx]
    artist_name = artist_rec["name"]
    artist_mbid = artist_rec["mbid"]
    artist_output.clear_output()
    with artist_output:
        display(Markdown(f"getting data..."))
    
    artist_info = lfm_get_artist_info(artist=artist_name, api_key=LFM_KEY, mbid=artist_mbid)
    do_artist = None
    if mbid:
        mb_artist = mb_get_artist_by_id(mbid)
        do_artist = discogs_get_artist_by_id(mb_get_discogs_id(mb_artist))

    sim_artists = lfm_get_similar_artists(artist=artist_name, api_key=LFM_KEY)
    bio = lfm_artist_bio(artist_info)
    new_artists, known_artists = get_new_sim_artists(top_artists_df, sim_artists)
    img_data = artist_rec.get("image", [])
    img_url = do_artist.data.get("thumb")
    if not img_url and img_data:
        if len(img_data) >= 2:
            img_url = img_data[1].get("#text", "")
        else:
            img_url = img_data[0].get("#text", "")

    artist_output.clear_output()
    with artist_output:
        display(Markdown(f"## {artist_name}"))
        display(Markdown(f"mbid: {artist_mbid}"))
        display(Markdown(bio.get("summary")))
        display(Image(url=img_url))
        display(Markdown(f"URL: {artist_rec['url']}"))
        if do_artist:
            display(Markdown("#### Aliases"))
            display(Markdown(", ".join([a.name for a in do_artist.aliases])))
            display(Markdown("#### Name variants"))
            display(Markdown(", ".join(do_artist.name_variations)))
        display(Markdown("#### Related Artists (new)"))
        display(Markdown(f"**{', '.join(new_artists)}**"))
        display(Markdown("#### Related Artists (known)"))
        display(Markdown(f"*{', '.join(known_artists)}*"))
    sel_new_art.options = new_artists
    sel_new_art.visible = bool(new_artists)
    move_to_selected_btn.visible = bool(new_artists)


def get_new_sim_artists(selected_artists: pd.DataFrame, sim_artists: dict):
    known_artists = set(selected_artists["name"].values)
    new_artists = sim_artists.keys() - known_artists
    known_artists = known_artists & sim_artists.keys()
    return new_artists, known_artists


def add_new_artists(btn_event):
    del btn_event
    for new_artist in sel_new_art.value:
        new_art_info = lfm_get_artist_info(artist=new_artist, api_key=LFM_KEY)
    new_artists[new_artist] = new_art_info
    all_new_art.options=sorted(new_artists.keys())


new_artists = dict()

artist_output = widgets.Output(layout={"border": "1px solid black"})
art_dict = {v: k for k,v in top_artists_df["name"].to_dict().items()}
selector = widgets.SelectMultiple(options=art_dict, 
                                  description="Select Artist",
                                  layout=widgets.Layout(width="50%", height="300px"),)
sel_new_art = widgets.SelectMultiple(options=[], 
                                     description="Select Artist",
                                     layout=widgets.Layout(width="300px", height="200px"),
                                     style={"description_width": "initial"},
                                     visible=False)
all_new_art = widgets.Select(options=[], 
                             description="Selected Artists",
                             layout=widgets.Layout(width="300px", height="300px"),
                             style={"description_width": "initial"})
add_btn = widgets.Button(description="Add to Tracked", visible=False)
sel_new_art.visible = False
add_btn.visible = False
add_btn.on_click(add_new_artists)

selector.observe(display_artist, names="value")
display(widgets.VBox([selector, artist_output,
                      widgets.HBox([widgets.VBox([sel_new_art, add_btn]), all_new_art])]))
artist_rec = top_artists_df.iloc[2]
artist_name = artist_rec["name"]
artist_mbid = artist_rec["mbid"]
do_artist = discogs_find_artist(artist_name)
sim_artists = lastfm_get_similar_artists(artist=artist_name, api_key=LAST_FM_KEY)
top_artists_df
last_fm_sim_artists = "http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={artist}&api_key={api_key}&format=json"

similar_artists = top_artists_df.set_index("mbid")[["name"]].to_dict()["name"]
for _, row in top_artists_df.iterrows():
    print(row["name"])
    resp = requests.get(last_fm_sim_artists.format(
        artist=row["name"],
        api_key="143c9cc433c0a23bb16ced806e8c40fb",
        limit=10
    ))
    sim_resp = resp.json()
    if "similarartists" not in sim_resp:
        continue
    for sim_artist in sim_resp["similarartists"]["artist"][:20]:
        if "mbid" in sim_artist:
            similar_artists[sim_artist["mbid"]] = sim_artist["name"]
similar_artists
# Remove unicode characters
import unicodedata
for artist_name in similar_artists.values():
    print(unicodedata.normalize('NFD', artist_name).encode('ascii', 'ignore').decode('utf-8'))
top_artists = top_artists_df.set_index("mbid")[["name"]].to_dict()["name"]
norm_top_artists = {normalize(a) for a in top_artists.values()}
releases2019[releases2019["Artist"].str.lower().isin(norm_top_artists)]
# import persistent
# from persistent.list import PersistentList
# from persistent.mapping import PersistentMapping
# from BTrees.OOBTree import OOBTree

# from dataclasses import dataclass, field
# from datetime import datetime
# from enum import Enum, auto
# import uuid
# from typing import Dict, List, Any


# def _parse_enum(enum_type, name: str):
#     cf_members = {n.casefold(): v for n,v in enum_type.__members__.items()}
#     if name.casefold() in cf_members:
#         return cf_members[name.casefold()]
#     return None

# class ArtistRating(Enum):
#     Other = auto()
#     Favorite = auto()
#     Like = auto()
#     DontLike = auto()
#     Unknown = auto()

#     @classmethod
#     def parse(cls, name: str):
#         enum_val = _parse_enum(cls, name)
#         return enum_val if enum_val else cls.Other

# class ArtistType(Enum):
#     Other = auto()
#     Person = auto()
#     Group = auto()

#     @classmethod
#     def parse(cls, name: str):
#         enum_val = _parse_enum(cls, name)
#         return enum_val if enum_val else cls.Other
    

# class RTArtists(persistent.Persistent):
    
#     def __init__(self):
#         self.artists: OOBTree = OOBTree()
#         self.artist_ids: OOBTree = OOBTree()
            
#     def add_artist(self, artist):
#         self.artists[artist.name] = artist
#         if artist.mbid:
#             self.artist_ids[artist.mbid] = artist

#     def del_artist(self, artist):
#         if isinstance(artist, RTArtist):
#             key = artist.name
#         elif isinstance(artist, str):
#             key = artist
            
#         if key in self.artists:
#             del_artist = self.artists.pop(key)
#             if del_artist.mbid in self.artist_ids:
#                 self.artist_ids.pop(del_artist.mbid)
#         elif key in self.artist_ids:
#             del_artist = self.artist_ids.pop(key)
#             if del_artist.name in self.artists:
#                 self.artists.pop(del_artist.name)

#     def __getitem__(self, key):
#         if key in self.artists:
#             return self.artists[key]
#         if key in self.artist_ids:
#             return self.artist_ids[key]
#         return None
        
#     def __contains__(self, key):
#         return key in self.artists or key in self.artist_ids
    
#     def add_artists(self, artists):
#         for artist in artists:
#             self.add_artist(artist)

# @dataclass
# class RTArtist(persistent.Persistent):
#     name: str
#     sort_name: str = None
#     mbid: str = None
#     discogs_id: str = None
#     artist_type: ArtistType = ArtistType.Other
#     artist_rank: int = 500
#     favorite: bool = False
#     lfm_url: str = None
#     discogs_url: str = None
#     bio_summary: str = None
#     bio_content: str = None
#     tags: PersistentList = field(default_factory=PersistentList)
#     artist_rating: ArtistRating = ArtistRating.Unknown
#     releases: OOBTree = field(default_factory=OOBTree)
#     aliases: PersistentList = field(default_factory=PersistentList)
#     name_variations: PersistentList = field(default_factory=PersistentList)
#     members: OOBTree = field(default_factory=OOBTree)
#     member_of: OOBTree = field(default_factory=OOBTree)
#     similar: OOBTree = field(default_factory=OOBTree)
#     ext_data: OOBTree = field(default_factory=OOBTree)
#     img_url: str = None
#     tracked: bool = False
    
#     @property
#     def musicbrainz_url(self):
#         if self.mbid:
#             return f"https://musicbrainz.org/artist/{self.mbid}"
#         return None
    
#     def has_data(self, key):
#         return key.casefold() in self.ext_data
    
#     def _set_ext_data(self, key, data, overwrite=True):
#         if key.casefold() in ["lfm_artist", "discogs_artist", "mb_artist"]:
#             self.ext_data[key.casefold()] = data
#         else:
#             raise KeyError(f"Unknown data type {key}")
    
#     def populate_discogs_data(self, overwrite=False):
#         if not overwrite and self.has_data("discogs_artist"):
#             return
#         if self.discogs_id:
#             artist = discogs_get_artist_by_id(self.discogs_id)
#         else:
#             artist = discogs_find_artist(self.name)
#         if artist:
#             self._set_ext_data("discogs_artist", artist)
#             self.discogs_url = f"https://www.discogs.com/{artist.url}"
#             self.aliases = PersistentList([a.name for a in artist.aliases])
#             self.name_variations = PersistentList(artist.name_variations)
#             if artist.images and "uri" in artist.images[0]:
#                 self.img_url = artist.images[0]["uri"] 
    
#     def populate_mb_data(self, overwrite=False):
#         if not overwrite and self.has_data("mb_artist"):
#             return
#         if not self.mbid:
#             return
#         artist = mb_get_artist_by_id(self.mbid)
#         if not artist:
#             return
#         self.artist_type = ArtistType.parse(artist.get("type"))
#         self.sort_name = artist.get("sort-name")
#         self.discogs_id = mb_get_discogs_id(artist)
#         self._set_ext_data("mb_artist", artist)
#         membership = mb_get_member_of(artist)
#         for member in membership:
#             member_artist = RTArtist(name = member["name"], mbid=member["id"])
#             if self.artist_type == ArtistType.Person:
#                 self.member_of[member_artist.name] = member_artist
#             elif self.artist_type == ArtistType.Group:
#                 self.members[member_artist.name] = member_artist

#     def populate_lfm_data(self, overwrite=False):
#         if not overwrite and self.has_data("lfm_artist"):
#             return
#         artist = lfm_get_artist_info(self.name, LFM_KEY)
#         if not artist:
#             return
#         print(list(self.ext_data.keys()))
#         self._set_ext_data("lfm_artist", artist)
#         self.mbid = artist.get("mbid")
#         self.lfm_url = artist.get("url")
#         bio = lfm_artist_bio(artist)
#         if bio:
#             self.bio_summary = bio.get("summary")
#             self.bio_content = bio.get("content")
#         tag_list = artist.get("tags", {}).get("tag", [])
#         tag_names = [t_item["name"] for t_item in tag_list]
#         self.tags.append(tag_names)

#     def get_similar_artists(self, artist_db=None, overwrite=False):
#         if not overwrite and self.similar:
#             return
#         similar_artists_raw = lfm_get_similar_artists(
#             artist=self.name, api_key=LFM_KEY, num_artists=40
#         )
#         similar_artists = {
#             name: s_artist for name, s_artist 
#             in similar_artists_raw.items() 
#             if float(s_artist["match"]) >= 0.125
#         }
#         for a_name, s_artist in similar_artists.items():
#             if artist_db:
#                 if a_name in artist_db.artists:
#                     print("linking existing artist:", a_name)
#                     self.similar[a_name] = artist_db[a_name]
#                     continue
#             self.similar[a_name] = RTArtist(
#                 name=a_name, mbid=s_artist.get("mbid"), lfm_url=s_artist.get("url"))

#     def _add_mb_releases(self):
#         if "mb_artist" in self.ext_data:
#             releases = mb_get_group_releases(self.ext_data["mb_artist"])
#             for release in releases:
#                 self.releases[release.title] = release

#     def _create_mb_release(self, release) -> RTRelease:
#         rel_date = release.get("first-release-date")
#         release = RTRelease(
#             title = release.get("title"),
#             mbid = release.get("id"),
#             primary_type = release.get("primary_type"),
#             other_type = release.get("type"),
#             release_date = rel_date if isinstance(rel_date, datetime) else None,
#         )
#         release.set_ext_data("mb_release", release)
#         return release

# @dataclass
# class RTRelease(persistent.Persistent):
#     title: str
#     mbid: str = None
#     primary_type: str = None
#     other_type: str = None
#     release_date: datetime = None
#     ext_data: OOBTree = field(default_factory=OOBTree)

#     def set_ext_data(self, key, data, overwrite=True):
#         if key.casefold() in ["lfm_release", "discogs_release", "mb_release"]:
#             self.ext_data[key.casefold()] = data
#         else:
#             raise KeyError(f"Unknown data type {key}")<jupyter_output><empty_output><jupyter_text>Filter based on normalized sets<jupyter_code>artist_names_norm = {art: set(normalize(art).split()) for art in top_artists.values()}
list(artist_names_norm.items())[:10]
artist_names_norm = {art: set(normalize(art).split()) for art in top_artists.values()}
for _, release in releases2019.iterrows():
    for t_artist in artist_names_norm.values():
        if t_artist and not (t_artist - set(normalize(release["Artist"]).split())):
            print(release["Album"], release["Artist"])
            print(t_artist, set(normalize(release["Artist"]).split()))
            break<jupyter_output>Gallipoli Beirut
{'beirut'} {'beirut'}
<NAME>
{'brown', 'ian'} {'brown', 'ian'}
No Geography The Chemical Brothers
{'brothers', 'the', 'chemical'} {'brothers', 'the', 'chemical'}
Social Cues Cage the Elephant
{'elephant', 'cage', 'the'} {'elephant', 'cage', 'the'}
Here Comes the Cowboy Mac DeMarco
{'mac', 'demarco'} {'mac', 'demarco'}
<NAME>! <NAME>
{'lana', 'del', 'rey'} {'lana', 'del', 'rey'}
Free Iggy Pop
{'pop', 'iggy'} {'pop', 'iggy'}
Beneath The Eyrie Pixies
{'pixies'} {'pixies'}
Giants of All Sizes Elbow
{'elbow'} {'elbow'}
La Linda Tei Shi
{'tei', 'shi'} {'tei', 'shi'}<jupyter_text>Ultimate filter!<jupyter_code>def matches_artist(release):
    for t_artist in artist_names_norm.values():
        if t_artist and not (t_artist - set(normalize(release["Artist"]).split())):
            return True
    return False
releases2019[releases2019.apply(matches_artist, axis=1)]
wiki_release_df[wiki_release_df.apply(matches_artist, axis=1)]<jupyter_output><empty_output><jupyter_text>Normalize data<jupyter_code>import unicodedata
def normalize(str):
    return unicodedata.normalize('NFD', str).encode('ascii', 'ignore').decode('utf-8').lower()
    
norm_fav_artists = {normalize(a) for a in similar_artists.values()}
for _, row in releases2019.iterrows():
    if row["Artist"].lower() in norm_fav_artists:
        print(row)
norm_fav_artists
releases2019[releases2019["Artist"].str.split(" ")]
releases2019["ArtistNames"] = releases2019.apply(lambda x: set(x["Artist"].split(" ")), axis=1)
top_artists_df[top_artists_df["name"].isin(releases2019["Artist"])]
import pandas as pd
import requests
from bs4 import BeautifulSoup


url = 'http://www.vru.gov.ua/act_list'


def get_
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
table = soup.find('table')

records = []
columns = []
for tr in table.findAll("tr"):
    ths = tr.findAll("th")
    if ths != []:
        for each in ths:
            columns.append(each.text)
    else:
        trs = tr.findAll("td")
        record = []
        for each in trs:
            try:
                link = each.find('a')['href']
                text = each.text
                record.append(link)
                record.append(text)
            except:
                text = each.text
                record.append(text)
        records.append(record)

columns.insert(1, 'Link')
df = pd.DataFrame(data=records, columns = columns)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12477144549457907, 'fraction_numerical': 0.010120779789343566, 'mean_word_length': 3.469098860628381, 'pattern_counts': {'":': 12, '<': 69, '<?xml version=': 0, '>': 77, 'https://': 6, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '581607', 'license': 'MIT', 'n_tokens_mistral': 13788, 'n_tokens_neox': 13398, 'n_words': 2269, 'path': 'DiscogLookup.ipynb', 'repo_name': 'ianhelle/discog-release', 'size': '38926'}