<commit_before>'''
Created on Oct 9, 2015

@author: akshah
'''
from bgpDataEngine.bgpDataEngine import bgpDataEngine
from customUtilities.helperFunctions import *


if __name__ == '__main__':
    start_time,_=current_time()
    
    bde=bgpDataEngine()
    
    #Fetch entire month
    #bde.getMonth('Updates','2014','03')

        
    #start='20150130000000'
    #end='20150131000100'
    #bde.getRangeFromBGPMon('updates',start,end)

    #Just load files no fetching
    #mrtFiles = [join('mrtFiles/', f) for f in listdir('mrtFiles/') if isfile(join('mrtFiles/', f)) if join('mrtFiles/',f)[-4:]=='.mrt'] 
    #bde.load2DB(mrtFiles)
    
    
    #bde.downloadFile()
    bde.getRange('updates','20140701','20140705',collectors=['route-views.jinx']) 
    
    #bde.load(['/raid1/akshah/mrtFiles/route-views.kixp/2015/11/01/route-views.kixp.updates.20151101.0000.bz2','/raid1/akshah/mrtFiles/route-views.kixp/2015/11/01/route-views.kixp.updates.20151101.1200.bz2'])
    #bde.load2DB()
    
    #Getting data from DB
    #day='20140705'
    #peer='196_223_14_10'
    #day='20140701'
    #peer='195_66_224_114'
    #bde.getUpdateMessages(peer,day)
    #while True:
    #    val=bde.messageQueue.get()
    #    if not val:
    #        break
    #    print(val)
    
    end_time,_=current_time()
    print('Finished processing in '+str(int((end_time-start_time)/60))+' minutes and '+str(int((end_time-start_time)%60))+' seconds.')<commit_msg>Simple main<commit_after>'''
Created on Oct 9, 2015

@author: akshah
'''
from bgpDataEngine.bgpDataEngine import bgpDataEngine
from customUtilities.helperFunctions import *


if __name__ == '__main__':
    start_time,_=currentTime()
    
    bde=bgpDataEngine()
    
    #Fetch entire month
    #bde.getMonth('Updates','2014','03')

        
    #start='20150130000000'
    #end='20150131000100'
    #bde.getRangeFromBGPMon('updates',start,end)

    #Just load files no fetching
    #mrtFiles = [join('mrtFiles/', f) for f in listdir('mrtFiles/') if isfile(join('mrtFiles/', f)) if join('mrtFiles/',f)[-4:]=='.mrt'] 
    #bde.load2DB(mrtFiles)
    
    
    #bde.downloadFile()
    #bde.getRange('updates','20140701','20140705',collectors=['route-views.jinx'])
    
    #bde.load(['/raid1/akshah/mrtFiles/route-views.kixp/2015/11/01/route-views.kixp.updates.20151101.0000.bz2','/raid1/akshah/mrtFiles/route-views.kixp/2015/11/01/route-views.kixp.updates.20151101.1200.bz2'])
    #bde.load2DB()
    
    #Getting data from DB
    #day='20140705'
    #peer='196_223_14_10'
    #day='20140701'
    #peer='195_66_224_114'
    #bde.getUpdateMessages(peer,day)
    #while True:
    #    val=bde.messageQueue.get()
    #    if not val:
    #        break
    #    print(val)
    
    end_time,_=currentTime()
    print('Finished processing in '+str(int((end_time-start_time)/60))+' minutes and '+str(int((end_time-start_time)%60))+' seconds.')
{'fraction_non_alphanumeric': 0.16585535465924894, 'fraction_numerical': 0.09944367176634214, 'mean_word_length': 3.588516746411483, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '576547', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1113, 'n_words': 150}