<commit_before>import sys
from xml.dom.minidom import parseString
from dicttoxml import dicttoxml
import string
import random
import shelve
import numpy as np 
import scipy.sparse as sparse
import cPickle as pickle
from collections import defaultdict, namedtuple

def randomString(length=16):
	return "".join([random.choice(string.letters) for _ in xrange(length)])

def token((disp, repr)):
	return {'disp':disp, 'repr':repr}

def randomText(length=30):
	return " ".join([random.choice(words) for _ in xrange(length)])

def remove_prefix(w):
	if '_' in w:
		return w.split('_', 1)[1]
	else:
		return w

words = file('/usr/share/dict/words').read().splitlines()
random.shuffle(words)
codeDict=pickle.load(file('codeDict.pk'))
words = words[:500]
vocab = set(words)

def randomPatient():
	global vocab
	pat = {}
	pat['index'] = randomString()
	ChiefComplaint = pat['ChiefComplaint'] = randomText(2)
	TriageAssessment = pat['TriageAssessment'] = randomText(10)
	MDcomments = pat['MDcomments'] = randomText(50)
	Age = pat['Age'] = str(np.random.choice(range(20,80)))
	Sex = pat['Sex'] = np.random.choice(['M', 'F'])[0]
        Diagnosis = {}
        code = np.random.choice(codeDict.keys())
        name = codeDict[code]
        Diagnosis['D_name'] = name
        Diagnosis['D_code'] = str(code.replace("code_", ""))
        for i in xrange(1,np.random.randint(5)):
            code = np.random.choice(codeDict.keys())
            name = codeDict[code]
            Diagnosis['D'+str(i)+'_name'] = name
            Diagnosis['D'+str(i)+'_code'] = str(code.replace("code_", ""))

        pat['Diagnosis'] = Diagnosis
        return {'visit':pat}

if __name__ == "__main__":
<commit_msg>removed dependency on ICD9 codes<commit_after>import sys
from xml.dom.minidom import parseString
from dicttoxml import dicttoxml
import string
import random
import shelve
import numpy as np 
import scipy.sparse as sparse
import cPickle as pickle
from collections import defaultdict, namedtuple

def randomString(length=16):
	return "".join([random.choice(string.letters) for _ in xrange(length)])

def token((disp, repr)):
	return {'disp':disp, 'repr':repr}

def randomText(length=30):
	return " ".join([random.choice(words) for _ in xrange(length)])

def remove_prefix(w):
	if '_' in w:
		return w.split('_', 1)[1]
	else:
		return w

words = file('/usr/share/dict/words').read().splitlines()
random.shuffle(words)
words = words[:500]
vocab = set(words)

def randomPatient():
	global vocab
	pat = {}
	pat['index'] = randomString()
	ChiefComplaint = pat['ChiefComplaint'] = randomText(2)
	TriageAssessment = pat['TriageAssessment'] = randomText(10)
	MDcomments = pat['MDcomments'] = randomText(50)
	Age = pat['Age'] = str(np.random.choice(range(20,80)))
	Sex = pat['Sex'] = np.random.choice(['M', 'F'])[0]
        return {'visit':pat}

if __name__ == "__main__":

{'fraction_non_alphanumeric': 0.15863524446007737, 'fraction_numerical': 0.014421385860007034, 'mean_word_length': 4.631683168316831, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1315705', 'n_tokens_mistral': 1018, 'n_tokens_neox': 962, 'n_words': 226}