<commit_before>import os, sys, operator
from helperFcns import *

version = "V7"
path = "/data3/mbc"                                                          #mbc path
oncoKey = path+"/oncoshare/oncoshare_s6_map.txt"                             #oncoshare-S6 map
noteMdata = path+"/notemetadata.txt"
#dictfile = "/data3/mbc/src/clever/mbc-dic.txt"
dictfile = "/data3/clever/mbc/mbc-dic_"+version+".txt"

#first neg, all else pos
testpts = ["1429499","1158149","1320506","298541","1429499","1619903","1838136","1523543"]

# main #
target_class = ["mbc","drecur","lrecur","mbcrx","recur","mets"]
s6pts = getS6ids(0,oncoKey)                                             #S6 ids for subset
termDict = getTerminology(dictfile)                                          #get terminology

#for pt in s6pts: print pt,s6pts[pt]
noteMDict = loadOncoNoteMdata(noteMdata)                                     #patient_id|note_id|doc_description|age_at_note_DATE_in_days|note_year

seqDict = {}
for target in target_class:
    seqFile = "/data3/clever/mbc/seq/"+version+"/"+target+"/extraction*.tsv"
    tmpDict = loadSeqs(seqFile,noteMDict,termDict)
    seqDict.update(tmpDict)
print len(seqDict)

for oncoid in s6pts:
    pt = str(s6pts[oncoid])
#for pt in testpts:
#    print pt
    ptAnts = []
    fout = open(path+"/oncoshare/clever/ptseq/"+version+"/seqs_"+pt+".txt","w") 
    for sid in seqDict:
        tmp = seqDict[sid]
        sinfo = tmp.split("|")
        #print sinfo
        if sinfo[4]==pt:
#            print "found"
            toff = sinfo[7]
            litem = [toff,tmp]
            if litem not in ptAnts:
                ptAnts.append(litem)
    ptList = sorted(ptAnts, key=operator.itemgetter(0))
    #tmpPtList = set(ptList)
    #ptList = list(tmpPtList)
    for item in ptList:
            #print item[0],item[1]
        print >> fout, item[1]
    fout.close()<commit_msg>updated the ptsequencing step for general use<commit_after>import os, sys, operator
from helperFcns import *

proj = "pqrs"
ppath = "../../proj/"+proj+"/"
ptKey = "../../proj/"+proj+"/ptkey.txt"  
noteMdata = "../../res/corpus/testnotemdata.txt"
dpath = "../../res/dicts/"
dictfile = dpath+"clever_terminology.txt"
# target classes #
target_class = ["mbc","drecur","lrecur","advp","recur","mets"]
s6pts = getPids(0,ptKey)                                             #S6 ids for subset
termDict = getTerminology(dictfile)                                          #get terminology

#for pt in s6pts: print pt,s6pts[pt]
noteMDict = loadOncoNoteMdata(noteMdata)                                     #patient_id|note_id|doc_description|age_at_note_DATE_in_days|note_year

seqDict = {}
for target in target_class:
    seqFile = ppath +"seq/"+target+"/extraction*.tsv"
    print seqFile
    tmpDict = loadSeqs(seqFile,noteMDict,termDict)
    seqDict.update(tmpDict)
print len(seqDict)

for oncoid in s6pts:
    pt = str(s6pts[oncoid])
#for pt in testpts:
    print pt
    ptAnts = []
    fout = open(ppath+"/ptseq/seqs_"+pt+".txt","w") 
    for sid in seqDict:
        tmp = seqDict[sid]
        sinfo = tmp.split("|")
        print sinfo
        if sinfo[4]==pt:
#            print "found"
            toff = sinfo[7]
            litem = [toff,tmp]
            if litem not in ptAnts:
                ptAnts.append(litem)
    ptList = sorted(ptAnts, key=operator.itemgetter(0))
    #tmpPtList = set(ptList)
    #ptList = list(tmpPtList)
    for item in ptList:
            #print item[0],item[1]
        print >> fout, item[1]
    fout.close()
{'fraction_non_alphanumeric': 0.13725490196078433, 'fraction_numerical': 0.02529127593066212, 'mean_word_length': 2.308270676691729, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6924797', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1224, 'n_words': 238}