<commit_before>import pymongo
from pymongo import MongoClient
from wowlib import wowapi, queries
import time


#connection informattion
client = MongoClient()

print ('Initiating db connection and getting wow data')
db = client.wow
print("retrieved data")
posts = db.auctiondata
itemdb = db.items



#create list of known users
knownitems = []
dbitems = itemdb.distinct('id')
for item in dbitems:
    knownitems.append(item)
    


#aggregates user name and server name return is {_id:{username:"",server:""}}





for item in posts.find().distinct('item'):
    try:
        


        if queries.binary_search(knownitems,item)== True:

            print ("already seen")
        else: 
            newitem = wowapi.itemquery(item)
            #print ('adding new player')
            itemdb.insert_one(newitem)
            newcount +=1
    except: 
        print("failed on tryblock")
<commit_msg>expanded from vscode<commit_after>import pymongo
from pymongo import MongoClient
from wowlib import wowapi, queries
import time


#connection informattion
client = MongoClient()

print ('Initiating db connection and getting wow data')
db = client.wow
print("retrieved data")
posts = db.auctiondata
itemdb = db.items



#create list of known users
knownitems = []
dbitems = itemdb.distinct('id')
for item in dbitems:
    knownitems.append(item)
    


#aggregates user name and server name return is {_id:{username:"",server:""}}





for item in posts.find().distinct('item'):
    try:
        


        if queries.binary_search(knownitems,item)== True:

            print ("already seen")
        else: 
            newitem = wowapi.itemquery(item)
            #print ('adding new player')
            itemdb.insert_one(newitem)
            newcount +=1
    except: 
        print("failed on tryblock yessir")

{'fraction_non_alphanumeric': 0.09944751381215469, 'fraction_numerical': 0.0011049723756906078, 'mean_word_length': 2.9714912280701755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7336300', 'n_tokens_mistral': 565, 'n_tokens_neox': 522, 'n_words': 163}