<commit_before>import pymongo
import sys
import os
from pymongo import MongoClient
#from makeConllFromDBOutput import makeConll
#from getJustSentences import getJustSentences

# python will convert \n to os.linesep

#Set size of training and test sets
sizeOfTraining = float(sys.argv[1])
if len(sys.argv) < 3:
	endSet = 1
else:
	endSet = float(sys.argv[2])

#print endSet, sizeOfTraining


client = MongoClient('mon-entity-event-r13-2.recfut.com:27016')
db = client.rf_entity_curation
allMalware = db.malware_copy
######################
# Open the databases we need to drop before making new ones.
testMalware = db.malware_test
bigPool = db.malware_training
startPool = db.malware_labeled
unlabeledPool = db.malware_unlabeled
testMalware.remove({})
bigPool.remove({})
startPool.remove({})
unlabeledPool.remove({})

########################
#Create databases to hold test and training sets.
#db.create_collection("malware_test")
#db.create_collection("malware_training")
#testMalware = db.malware_test
#bigPool = db.malware_training
# trainingCollection = allMalware.find({"random" : { "$gt": 0, "$lt": sizeOfTraining }})
# testCollection = allMalware.find({"random" : { "$gt": sizeOfTraining, "$lt": endSet }})
# bigPool.insert(trainingCollection)
# testMalware.insert(testCollection)
#
#
#
# ########################
# # Divide the training set into a small labeled pool for the model to start on
# # and a bigger unlabeled pool which we will train the model with.
# sizeOfstartPool = 0.05
# tmpStartPool = bigPool.find({"random" : { "$gt": 0, "$lt": sizeOfstartPool}})
# tmpUnlabeledPool = bigPool.find({"random" : { "$gt": sizeOfstartPool, "$lt": 1 }})
# startPool = db.malware_labeled
# unlabeledPool = db.malware_unlabeled
# startPool.insert(tmpStartPool)
# unlabeledPool.insert(tmpUnlabeledPool)


############################
# Create a labeledPool and an unlabeledPool file to which we write the data of the labeled pool collection
# Make this text file into a conll file using makeConll. 
# # This will later be used to train epic.
# tmpStartPool = startPool.find({"random" : { "$gt": 0, "$lt": 1}})
# f = open(os.path.expanduser("~/epic/epic/data/labeledPool.txt"),'w')
# #f = open("labeledPool.txt",'w')
# #stringarray = []
# for i in range(1,tmpStartPool.count()):
# 	f.write(str(tmpStartPool[i]))
# 	#	stringarray.append(tmpStartPool[i])
# 	f.write("\n")
#
# f.close()
# tmpUnlabeledPool = unlabeledPool.find({"random" : { "$gt": 0, "$lt": 1}})
# f = open(os.path.expanduser("~/epic/epic/data/unlabeledPool.txt"),'w')
# #f = open("unlabeledPool.txt",'w')
# for i in range(1,tmpUnlabeledPool.count()):
# 	f.write(str(tmpUnlabeledPool[i]))
# 	f.write("\n")
#
# f.close()



#tmp_file = open(os.path.expanduser('~/epic/epic/data/labeledPool.conll'))
#tmp_file.close()

#makeConll('~/epic/epic/data/labeledPool.txt', '~/epic/epic/data/labeledPool.conll')

print "poop"
<commit_msg>Python script updated opefully tester calls python correctly<commit_after>import pymongo
import sys
import os
from pymongo import MongoClient
from makeConllFromDBOutput import makeConll
#from getJustSentences import getJustSentences

# python will convert \n to os.linesep

#Set size of training and test sets
sizeOfTraining = float(sys.argv[1])
if len(sys.argv) < 3:
	endSet = 1
else:
	endSet = float(sys.argv[2])

#print endSet, sizeOfTraining


client = MongoClient('mon-entity-event-r13-2.recfut.com:27016')
db = client.rf_entity_curation
allMalware = db.malware_copy
######################
# Open the databases we need to drop before making new ones.
testMalware = db.malware_test
bigPool = db.malware_training
startPool = db.malware_labeled
unlabeledPool = db.malware_unlabeled
testMalware.remove({})
bigPool.remove({})
startPool.remove({})
unlabeledPool.remove({})

########################
#Create databases to hold test and training sets.
#db.create_collection("malware_test")
#db.create_collection("malware_training")
#testMalware = db.malware_test
#bigPool = db.malware_training
trainingCollection = allMalware.find({"random" : { "$gt": 0, "$lt": sizeOfTraining }})
testCollection = allMalware.find({"random" : { "$gt": sizeOfTraining, "$lt": endSet }})
bigPool.insert(trainingCollection)
testMalware.insert(testCollection)



########################
# Divide the training set into a small labeled pool for the model to start on
# and a bigger unlabeled pool which we will train the model with.
sizeOfstartPool = 0.05
tmpStartPool = bigPool.find({"random" : { "$gt": 0, "$lt": sizeOfstartPool}})
tmpUnlabeledPool = bigPool.find({"random" : { "$gt": sizeOfstartPool, "$lt": 1 }})
startPool = db.malware_labeled
unlabeledPool = db.malware_unlabeled
startPool.insert(tmpStartPool)
unlabeledPool.insert(tmpUnlabeledPool)


############################
# Create a labeledPool and an unlabeledPool file to which we write the data of the labeled pool collection
# Make this text file into a conll file using makeConll. 
# # This will later be used to train epic.
tmpStartPool = startPool.find({"random" : { "$gt": 0, "$lt": 1}})
f = open(os.path.expanduser("~/epic/epic/data/labeledPool.txt"),'w')
#f = open("labeledPool.txt",'w')
#stringarray = []
for i in range(1,tmpStartPool.count()):
	f.write(str(tmpStartPool[i]))
	#	stringarray.append(tmpStartPool[i])
	f.write("\n")

f.close()
tmpUnlabeledPool = unlabeledPool.find({"random" : { "$gt": 0, "$lt": 1}})
f = open(os.path.expanduser("~/epic/epic/data/unlabeledPool.txt"),'w')
#f = open("unlabeledPool.txt",'w')
for i in range(1,tmpUnlabeledPool.count()):
	f.write(str(tmpUnlabeledPool[i]))
	f.write("\n")

f.close()



tmp_file = open(os.path.expanduser('~/epic/epic/data/labeledPool.conll'))
tmp_file.close()

makeConll('~/epic/epic/data/labeledPool.txt', '~/epic/epic/data/labeledPool.conll')

print "poop"

{'fraction_non_alphanumeric': 0.17731851337269885, 'fraction_numerical': 0.008336227856894755, 'mean_word_length': 5.905275779376499, 'pattern_counts': {'":': 24, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3228182', 'n_tokens_mistral': 2062, 'n_tokens_neox': 1944, 'n_words': 461}