<jupyter_start><jupyter_text>Background data 1 million - TrackLevel Features'''track-only features:Double: number of background playlists in which this track appears / total number of background playlistsDouble: number of background playlists in which this artist appears / total number of background playlistsDouble: number of background playlists in which this album appears / total number of background playlistsBinary: track title contains words like “remix”, “remixed”, “remixes”Double: number of background playlists in which this track appears where playlist list title contains words like “top”, “best”, “popular”, “hot”, “hits”, “🔥” / total number of background playlistsDouble: number of background playlists in which this track appears where playlist list title contains words like “latest”, “new”, “recent” / total number of background playlistsDouble: number of background playlists in which this track appears where playlist list title contains words like “remix”, “remixed”, “remixes” / total number of background playlists'''<jupyter_code>import itertools as it
import sys
import json
import re
import collections
import os
import pickle
from collections import Counter
from collections import defaultdict

count=0
path='../../data'
filenames = os.listdir(path)
backgroundPids=pickle.load(open('./SplitsInformation/backgroundDataPids.pkl','rb'))

popWords=['top', 'best', 'popular', 'hot', 'hits', 'fav', 'lit']
recencyWords=['latest', 'new', 'recent']
remixWords=['remix', 'remixed', 'remixes', 'mix']

trackCount=defaultdict(int)#
trackPopCount=defaultdict(int)#
trackRecencyCount=defaultdict(int)#
trackRemixCount=defaultdict(int)#
artistCount=defaultdict(int)#
albumCount=defaultdict(int)#
trackArtistAlbum={}#


for filename in sorted(filenames):
    if filename.startswith("mpd.slice.") and filename.endswith(".json"):
        fullpath = os.sep.join((path, filename))
        f = open(fullpath)
        js = f.read()
        f.close()
        mpd_slice = json.loads(js)
        for playlist in mpd_slice['playlists']:
                playlistId=str(playlist['pid'])
            
                playlistTitle=playlist['name']
                normPlaylistTitle=normalize_nameTitle(playlistTitle)
                pop=False
                for word in popWords:
                    if word in normPlaylistTitle.split():
                        pop=True
                
                recency=False
                for word in recencyWords:
                    if word in normPlaylistTitle.split():
                        recency=True
                        
                remix=False
                for word in remixWords:
                    if word in normPlaylistTitle.split():
                        remix=True
                        
                artists=[]
                albums=[]
                for track in playlist['tracks']:
                    trackId=track['track_uri'].replace('spotify:track:','')
                    
                    if trackId not in trackArtistAlbum:
                        trackTitle=track['track_name']
                        normTrackTitle=normalize_nameTitle(trackTitle)
                        trackRemix=False
                        for word in remixWords:
                            if word in normTrackTitle.split():
                                trackRemix=True
                        trackArtistAlbum[trackId]=(track['artist_uri'],track['album_uri'],trackRemix&1, normTrackTitle)
                    
                    trackCount[trackId]+=1
                    if pop:
                        trackPopCount[trackId]+=1
                    if recency:
                        trackRecencyCount[trackId]+=1
                    if remix:
                        trackRemixCount[trackId]+=1
                    
                    artists.append(track['artist_uri'])
                    albums.append(track['album_uri'])
                
                for art in set(artists):
                    artistCount[art]+=1
                for alb in set(albums):
                    albumCount[alb]+=1
                
                count=count+1
                if count % 10000 ==0:
                    print 'processed' + str(count)
import pickle
trackCount=pickle.load(open('./trackCountAllData.pickle','rb'))
trackPopCount=pickle.load(open('./trackPopCountAllData.pickle','rb'))
trackRecencyCount=pickle.load(open('./trackRecencyCountAllData.pickle','rb'))
trackRemixCount=pickle.load(open('./trackRemixCountAllData.pickle','rb'))
artistCount=pickle.load(open('./artistCountAllData.pickle','rb'))
albumCount=pickle.load(open('./albumCountAllData.pickle','rb'))
trackArtistAlbum=pickle.load(open('./trackArtistAlbumAllData.pickle','rb'))
numBGPlaylists=1000000
'''
track-only features:
Double: number of background playlists in which this track appears# / total number of background playlists ##
Double: number of background playlists in which this artist appears# / total number of background playlists ##
Double: number of background playlists in which this album appears# / total number of background playlists ##
Binary: track title contains words like “remix”, “remixed”, “remixes”##

Double: number of background playlists in which this track appears where playlist list title contains words like “top”, “best”, “popular”, “hot”, “hits”, “🔥” / total number of background playlists
Double: number of background playlists in which this track appears where playlist list title contains words like “latest”, “new”, “recent” / total number of background playlists
Double: number of background playlists in which this track appears where playlist list title contains words like “remix”, “remixed”, “remixes” / total number of background playlists
trackPopCount=defaultdict(int)#
trackRecencyCount=defaultdict(int)#
trackRemixCount=defaultdict(int)#
artistCount=defaultdict(int)#
albumCount=defaultdict(int)#
trackArtistAlbum={}#
'''
numBGPlaylists=1000000
trackFeatures={}

for trackId, count in trackCount.items():
    trackArtist=trackArtistAlbum[trackId][0]
    trackAlbum=trackArtistAlbum[trackId][1]
    
    countRatio=count*1.0/numBGPlaylists
    artistRatio=artistCount[trackArtist]*1.0/numBGPlaylists
    albumRatio=albumCount[trackAlbum]*1.0/numBGPlaylists
    isRemixTrack=trackAlbum=trackArtistAlbum[trackId][2]
    popRatio=trackPopCount[trackId]*1.0/numBGPlaylists
    recencyRatio=trackRecencyCount[trackId]*1.0/numBGPlaylists
    remixRatio=trackRemixCount[trackId]*1.0/numBGPlaylists
    
    trackFeatures[trackId]=(countRatio,artistRatio,albumRatio,isRemixTrack,popRatio,recencyRatio,remixRatio)
len(trackFeatures.items())
#For playlists in training data identify most frequent artists, albums etc over the revealed playlist
errorTracks=[]
playlistCounters={}
pidErrors=[]
for pid in QEPRFOp:
    revealedTracks=QEPRFOp[pid][0]
    #print revealedTracks
    artists=[]
    albums=[]
    for track in revealedTracks:
        try:
            artists.append(trackArtistAlbum[track][0])
            albums.append(trackArtistAlbum[track][1])
        except:
            errorTracks.append(track)
            pidErrors.append(pid)
    artistCounter=Counter(artists)
    albumCounter=Counter(albums)
    #print artists, artistCounter
    #print albums, albumCounter
    numUniqueAlbums=len(albumCounter)
    numUniqueArtists=len(artistCounter)
    #print numUniqueAlbums
    #print numUniqueArtists
    freqAlbumCount=albumCounter.most_common(1)[0][1]
    freqArtistCount=artistCounter.most_common(1)[0][1]
    #print freqAlbumCount
    #print freqArtistCount
    playlistCounters[pid]=[artists, albums,artistCounter,albumCounter,numUniqueAlbums,numUniqueArtists,freqAlbumCount,freqArtistCount]<jupyter_output><empty_output><jupyter_text>ChallengeData - Playlist Level Features'''Playlist-only features:Binary: is the input playlist title availableInteger: number of tracks in input playlistInteger: number of heldout tracks in input playlistDouble: number of unique albums / number of tracks in input playlist Double: number of unique artists / number of tracks in input playlist Double: frequency of most frequent album / number of tracks in input playlist Double: frequency of most frequent artist / number of tracks in input playlist Binary: playlist title contains words like “top”, “best”, “popular”, “hot”, “hits”, “🔥”Binary: playlist title contains words like “latest”, “new”, “recent”Binary: playlist title contains words like “remix”, “remixed”, “remixes”'''<jupyter_code>'''
    {
            "name": "Mama", 
            "num_holdouts": 28, 
            "pid": 1006778, 
            "num_tracks": 29, 
            "tracks": [
                {
                    "pos": 0, 
                    "artist_name": "<NAME>", 
                    "track_uri": "spotify:track:0NiXXAI876aGImAd6rTj8w", 
                    "artist_uri": "spotify:artist:1HBjj22wzbscIZ9sEb5dyf", 
                    "track_name": "Mama", 
                    "album_uri": "spotify:album:0zMLyv1kNV2B0LDGEK2RbB", 
                    "duration_ms": 181614, 
                    "album_name": "<NAME>: Electronic Nature - The Mix 2017"
                }
            ], 
            "num_samples": 1
    '''
import itertools as it
import sys
import json
import re
import collections
import os
import pickle
from collections import Counter

count=0

popWords=['top', 'best', 'popular', 'hot', 'hits', 'fav', 'lit']
recencyWords=['latest', 'new', 'recent']
remixWords=['remix', 'remixed', 'remixes', 'mix']

playlistFeatures={}

filename = '/home/ubuntu/testData/challenge_set.json'
f = open(filename)
js = f.read()
f.close()
mpd_slice = json.loads(js)


for playlist in mpd_slice['playlists']:
    playlistId=str(playlist['pid'])
    playlistMeta=testPidMeta[playlistId] #(testType,title,qtype,listLen)
    playlistTitle=''
    normPlaylistTitle=''
    if 'name' in playlist:
        playlistTitle=playlist['name']
        normPlaylistTitle=normalize_nameTitle(playlistTitle)
    numTracks=playlist['num_tracks']

    artists=[]
    albums=[]
    artistCounter={}
    albumCounter={}
    numUniqueAlbums=0
    numUniqueArtists=0
    freqAlbumCount=0
    freqArtistCount=0
    if playlistId in playlistCounters:
        #[artists, albums,artistCounter,albumCounter,numUniqueAlbums,numUniqueArtists,freqAlbumCount,freqArtistCount]
        artists=playlistCounters[playlistId][0]
        albums=playlistCounters[playlistId][1]
        artistCounter=playlistCounters[playlistId][2]
        albumCounter=playlistCounters[playlistId][3]
        numUniqueAlbums=playlistCounters[playlistId][4]
        numUniqueArtists=playlistCounters[playlistId][5]
        freqAlbumCount=playlistCounters[playlistId][6]
        freqArtistCount=playlistCounters[playlistId][7]

    titlePresent=playlistMeta[1]

    numRevealed=playlistMeta[-1]
    numHeldOut=numTracks-numRevealed

    albumRatio=numUniqueAlbums*1.0/numTracks
    artistRatio=numUniqueArtists*1.0/numTracks

    freqAlbumRatio=freqAlbumCount*1.0/numTracks
    freqArtistRatio=freqArtistCount*1.0/numTracks

    pop=False
    for word in popWords:
        if word in normPlaylistTitle.split():
            pop=True

    recency=False
    for word in recencyWords:
        if word in normPlaylistTitle.split():
            recency=True

    remix=False
    for word in remixWords:
        if word in normPlaylistTitle.split():
            remix=True

    playlistFeatures[playlistId]=(titlePresent&1, numTracks, numRevealed, numHeldOut, albumRatio, artistRatio, freqAlbumRatio, freqArtistRatio, pop&1, recency&1, remix&1, normPlaylistTitle, artistCounter, albumCounter)
    count=count+1
    if count % 1000 ==0:
        print 'processed' + str(count)
playlistFeatures.items()[1000:1050]
#generate ranks from scores -> takes ties into consideration
import pandas as pd
def getRanks(tupledOp):
    trackRanks={}
    if len(tupledOp)>0:
        df = pd.DataFrame.from_records(tupledOp)
        df['rank'] = df[2].rank(method='min', ascending=False)
        rankedList = list(df[[1, 'rank']].to_records(index=False))
        for track, rank in rankedList:
            trackRanks[track]=rank
    return trackRanks
#for each prediction for a playlist, get ranks
import random
pidSampleRanks={}
count=0
#(testType,title,qtype,listLen)
for pid, pidMeta in testPidMeta.items():
    considerMeta1Meta2=pidMeta[1]
    considerQE=True
    if pidMeta[2]=='TitleOnly':
        considerQE=False
    pidTracks=[]
    if considerQE:
        pidTracks=[x.replace('spotify:track:','') for x in pidTrackMapping[pid]]
    
    revealedTracks=set()
    predTracksMeta1=[]
    predTracksMeta2=[]
    predTracksQE=[]
    trackRanksQE={}
    trackRanksMeta1={}
    trackRanksMeta2={}
    
    if considerQE:
        QE=QEPRFOp[pid]
        revealedTracks=set(QE[0])
        validQE1= [x for x in QE[1] if x[1] not in revealedTracks]
        predTracksQE=[x[1] for x in validQE1][:1000]
        trackRanksQE=getRanks(validQE1)
    if considerMeta1Meta2 and pid in Meta1Op:
        Meta1=Meta1Op[pid]
        Meta2=Meta2Op[pid]
        validMeta1=[x for x in Meta1 if x[1] not in revealedTracks]
        validMeta2=[x for x in Meta2 if x[1] not in revealedTracks]
        predTracksMeta1=[x[1] for x in validMeta1][:500]
        predTracksMeta2=[x[1] for x in validMeta2][:500]
        trackRanksMeta1=getRanks(validMeta1)
        trackRanksMeta2=getRanks(validMeta2)
    
    allCandidateTracks=set(predTracksMeta1+predTracksMeta2+predTracksQE)
    positiveSamples=list(allCandidateTracks)
    negativeSamples=[]
    
    AllSamples=positiveSamples+negativeSamples
    
    positiveRanks=[]
    negativeRanks=[]
    for sample in AllSamples:
        if sample in predTracksQE:
            QErank= trackRanksQE[sample]
        else:
            #if track not part of prediction set, give a high number
            QErank=1050
        if sample in predTracksMeta1:
            Meta1rank= trackRanksMeta1[sample]
        else:
            Meta1rank=550
        if sample in predTracksMeta2:
            Meta2rank= trackRanksMeta2[sample]
        else:
            Meta2rank=550
        
        if sample in positiveSamples:
            positiveRanks.append((sample, QErank, Meta1rank, Meta2rank))
        else:
            negativeRanks.append((sample, QErank, Meta1rank, Meta2rank))
    pidSampleRanks[pid]=(positiveRanks,negativeRanks)
    count=count+1
    if count % 1000 ==0:
        print 'processed' + str(count)
len(pidSampleRanks)
count=0
for pid in pidSampleRanks:
    count=count+len(pidSampleRanks[pid][0])+len(pidSampleRanks[pid][1])
count
#putting all together
import sys
def formatFloat(val):
    if val==0:
        return str(0)
    return format(val, '.10f')

count=0
TrainingLines=[]
for pid, trainingSamples in pidSampleRanks.items():
    pidFeatures=playlistFeatures[pid]
    #print pidFeatures[:-3]
    posSamples=[x[0] for x in trainingSamples[0]]
    negSamples=[x[0] for x in trainingSamples[1]]
    AllSamples=trainingSamples[0]+trainingSamples[1]
    for sample in AllSamples:
        track=sample[0]
        trackMeta=trackArtistAlbum[track]
        trFeatures=trackFeatures[track]
        trArtist=trFeatures[0]
        trAlbum= trFeatures[1]
        artistCounter= pidFeatures[-2]
        albumCounter= pidFeatures[-1]
        
        trArtistCount=0
        if trArtist in artistCounter:
            trArtistCount=artistCounter[trArtist]
        
        trAlbumCount=0
        if trAlbum in albumCounter:
            trAlbumCount=albumCounter[trAlbum]
        
        
        if track in posSamples:
            label=1
        else:
            label=0
        Qid=pid
        QERank= sample[1]+1
        Meta1Rank= sample[2]+1
        Meta2Rank= sample[3]+1
        titleAvailable= pidFeatures[0]
        numTracks= pidFeatures[1]
        numRevealed= pidFeatures[2]
        numHeldOut= pidFeatures[3]
        albumRatio= formatFloat(pidFeatures[4])
        artistRatio= formatFloat(pidFeatures[5])
        freqAlbumRatio= formatFloat(pidFeatures[6])
        freqArtistRatio= formatFloat(pidFeatures[7])
        if titleAvailable:
            pop= pidFeatures[8]
            rec= pidFeatures[9]
            rem= pidFeatures[10]
        else:
            pop= 0
            rec= 0
            rem= 0
        trackcountRatio=formatFloat(trFeatures[0])
        trackartistRatio=formatFloat(trFeatures[1])
        trackalbumRatio=formatFloat(trFeatures[2])
        isRemixTrack=trFeatures[3]
        
        trArtistCountRatio= formatFloat(trArtistCount*1.0/numTracks)
        trAlbumCountRatio= formatFloat(trAlbumCount*1.0/numTracks)
        
        trpopRatio= formatFloat(trFeatures[4])
        trrecencyRatio=formatFloat(trFeatures[5])
        trremixRatio=formatFloat(trFeatures[6])
        
        TrainingLines.append([track,label, Qid, QERank, Meta1Rank, Meta2Rank, titleAvailable, numTracks, numRevealed, numHeldOut, albumRatio, artistRatio, freqAlbumRatio, freqArtistRatio, pop, rec, rem, trackcountRatio, trackartistRatio, trackalbumRatio, isRemixTrack, trArtistCountRatio, trAlbumCountRatio, trpopRatio, trrecencyRatio, trremixRatio])
    count=count+1
    if count % 1000 ==0:
        print 'processed' + str(count)
len(pidSampleRanks)
len(TrainingLines)
opLines=[]
for line in TrainingLines:
    trackId=line[0]
    op=''
    op=op+str(line[1])+' qid:'+line[2]
    count=1
    for val in line[3:]:
        op=op+' '+str(count)+':'+str(val)
        count=count+1
    op=op+' #'+trackId
    opLines.append(op)
print len(opLines)
opLines[95]
opTrainingFile='./Training/ChallengeSetFeatureFile.txt'
with open(opTrainingFile, 'w') as file_handler:
    for line in opLines:
        file_handler.write(line+'\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09984170058796925, 'fraction_numerical': 0.016395296246042516, 'mean_word_length': 3.0786257781876873, 'pattern_counts': {'":': 14, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '165538', 'license': 'Apache-2.0', 'n_tokens_mistral': 5490, 'n_tokens_neox': 5246, 'n_words': 1184, 'path': 'notebooks/CreateRerankingFeaturesChallengeSetAllData.ipynb', 'repo_name': 'skallumadi/BachPropagate', 'size': '17696'}