<commit_before>__author__ = 'Ryan'
import time;
import sys;
from pymongo import MongoClient

print("Processing Matches into Kill Collection");

client = MongoClient('mongo',27017)
db = client.urfday

def processMatchesIntoKillCollection():
    print("Adding matches to our kill Collection")
    matchCollection = db.matches
    KillCollection=db.totalKillCollection
    ProcessMatchCursor=db.ProcessMatchCursor;
    try:
        idToStart = ProcessMatchCursor.find()[0]['LastRun']
        matchIdToDelete = matchCollection.find({'_id':idToStart})[0]['matchId']
        print("Deleting match ID " + str(matchIdToDelete) + " for being incomplete ")
    except:
        KillCollection.drop()
        print("We have not processed any games, starting from the beginning!");
        idToStart = matchCollection.find().sort('_id',1)[0]['_id']
        print("New starting _id is " + str(idToStart));
    else:
        try:
            KillCollection.remove({'MatchId':matchIdToDelete})
        except:
            print(" We decided to ignore this case, Continuing with same collection without dropping");


    matchCursor = matchCollection.find({'_id':{'$gte':idToStart}}).sort('_id',1)
    #matchCursor.add_option(2)



    print("Resuming procssing at ID " + str(idToStart) )
    #for i in range(0,matchCollection.count()):
    i = 0;
    while matchCursor.alive:
        i = i + 1
        try:
            matchData = matchCursor.next()
        except StopIteration:
            #print("Nothing else to be done, sleeping")
            time.sleep(1);
            i = i -1
            continue;

        nextID = matchData['_id']
        #print("NEXTID is " + str(nextID))
        #except:
        #    print("Nothing else to be done, sleeping")
        #    time.sleep(100);
        #    i = i -1

         #   continue;
        print ("Saving ID " + str(nextID))
        previousRunID = ProcessMatchCursor.find()[0]['_id']
        ProcessMatchCursor.insert({'LastRun':nextID})
        ProcessMatchCursor.remove({'_id':previousRunID})

        if(i%10 == 0):
            sys.stdout.write("    " + str(KillCollection.count()) +"/"+ str(matchCollection.count()) + " Matches Analyzed\r")
            if(i%100 != 0):
                sys.stdout.flush()



        matchId = matchData['matchId']
        #print(matchData[i]['matchId'])
        participants = matchData['participants']
        ParticipantIDs = [-1]*11
        LastSeasonRankFromIDs = [-1]*11

        # Bind ParticipantIds to their respective Champions
        # Note Ids are 1-10, 0 is reserved for minions and should be left as -1 championId
        for j in range (0,len(participants)):

            ParticipantIDs[participants[j]['participantId']] = participants[j]['championId']
            LastSeasonRankFromIDs[participants[j]['participantId']] = participants[j]['highestAchievedSeasonTier']

        #print (ParticipantIDs)
        try:
            timeline = matchData['timeline']
            if(timeline['frameInterval'] != 60000):
                # throw an error, We don't expect data that isn't at 1 minute margins
                raise Exception('Wrong_Frame_Interval')

            #for every frame in the game
            gameFrames = timeline['frames']
        except:
            break;

        PlayerItems = [];
        for item in range(0,11):
            ItemList = []
            PlayerItems.append(ItemList)<commit_msg>changed break -> continue so loop stops repeating forever if there is no timeline<commit_after>__author__ = 'Ryan'
import time;
import sys;
from pymongo import MongoClient

print("Processing Matches into Kill Collection");

client = MongoClient('mongo',27017)
db = client.urfday

def processMatchesIntoKillCollection():
    print("Adding matches to our kill Collection")
    matchCollection = db.matches
    KillCollection=db.totalKillCollection
    ProcessMatchCursor=db.ProcessMatchCursor;
    try:
        idToStart = ProcessMatchCursor.find()[0]['LastRun']
        matchIdToDelete = matchCollection.find({'_id':idToStart})[0]['matchId']
        print("Deleting match ID " + str(matchIdToDelete) + " for being incomplete ")
    except:
        KillCollection.drop()
        print("We have not processed any games, starting from the beginning!");
        idToStart = matchCollection.find().sort('_id',1)[0]['_id']
        print("New starting _id is " + str(idToStart));
    else:
        try:
            KillCollection.remove({'MatchId':matchIdToDelete})
        except:
            print(" We decided to ignore this case, Continuing with same collection without dropping");


    matchCursor = matchCollection.find({'_id':{'$gte':idToStart}}).sort('_id',1)
    #matchCursor.add_option(2)



    print("Resuming procssing at ID " + str(idToStart) )
    #for i in range(0,matchCollection.count()):
    i = 0;
    while matchCursor.alive:
        i = i + 1
        try:
            matchData = matchCursor.next()
        except StopIteration:
            #print("Nothing else to be done, sleeping")
            time.sleep(1);
            i = i -1
            continue;

        nextID = matchData['_id']
        #print("NEXTID is " + str(nextID))
        #except:
        #    print("Nothing else to be done, sleeping")
        #    time.sleep(100);
        #    i = i -1

         #   continue;
        print ("Saving ID " + str(nextID))
        previousRunID = ProcessMatchCursor.find()[0]['_id']
        ProcessMatchCursor.insert({'LastRun':nextID})
        ProcessMatchCursor.remove({'_id':previousRunID})

        if(i%10 == 0):
            sys.stdout.write("    " + str(KillCollection.count()) +"/"+ str(matchCollection.count()) + " Matches Analyzed\r")
            if(i%100 != 0):
                sys.stdout.flush()



        matchId = matchData['matchId']
        #print(matchData[i]['matchId'])
        participants = matchData['participants']
        ParticipantIDs = [-1]*11
        LastSeasonRankFromIDs = [-1]*11

        # Bind ParticipantIds to their respective Champions
        # Note Ids are 1-10, 0 is reserved for minions and should be left as -1 championId
        for j in range (0,len(participants)):

            ParticipantIDs[participants[j]['participantId']] = participants[j]['championId']
            LastSeasonRankFromIDs[participants[j]['participantId']] = participants[j]['highestAchievedSeasonTier']

        #print (ParticipantIDs)
        try:
            timeline = matchData['timeline']
            if(timeline['frameInterval'] != 60000):
                # throw an error, We don't expect data that isn't at 1 minute margins
                raise Exception('Wrong_Frame_Interval')

            #for every frame in the game
            gameFrames = timeline['frames']
        except:
            continue;

        PlayerItems = [];
        for item in range(0,11):
            ItemList = []
            PlayerItems.append(ItemList)
{'fraction_non_alphanumeric': 0.12025040034939584, 'fraction_numerical': 0.014266996651623235, 'mean_word_length': 2.5285053929121726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3144520', 'n_tokens_mistral': 2032, 'n_tokens_neox': 1896, 'n_words': 467}