<commit_before>import pickle,sys,socket,subprocess,os,git,random
from pymongo import MongoClient
from datetime import datetime
from time import mktime,sleep
#from planetlab import Monitor
from configobj import ConfigObj


try:
    import json
except ImportError:
    import simplejson as json

PLE_CONF_FILE='/etc/ple.conf'
TEJO_CONF_FILE='/etc/tejo.conf'



def clone_git_repository(uri_repository):
    subprocess.Popen(['git','clone',uri_repository], stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip()

def getRTT_SSH(hostname, yanoama_root):
    try:
        script_to_run=yanoama_root+'/yanoama/monitoring/get_rtt_ssh.sh'
        rtt=float(subprocess.Popen(['sh',script_to_run,hostname], stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip())
        if rtt>0.0:
            return rtt
        else:
            return -1
    except:
        return -1

#this returns a list of peers [ {k1:e1,...},...]
#each entry the the following structure:
#{u'rtt': 11.0, u'_id': ObjectId('55b955cf465cba3f890fa36e'), 
#  u'server': u'us-monitor-001.cloudapp.net', 
#  u'client': u'planetlab1.cs.ucla.edu'}
def get_list_of_peers():
    host=(socket.gethostname())
    c = MongoClient(host, 27017)
    db=c.tejo
    status=db.status
    list_of_peers=list(status.find())
    c.close()    
    return list_of_peers

def update_peer_monitor(peer, monitor, rtt,monitors):
    host=(socket.gethostname())
    c = MongoClient(host, 27017)
    db=c.tejo
    status=db.status
    status.update({'peer':peer},{'$set':{'monitor':monitor}}, upsert=False,check_keys=False)
    status.update({'peer':peer},{'$set':{'monitor_rtt':rtt}}, upsert=False,check_keys=False)
    status.update({'peer':peer},{'$set':{'monitors':monitors}}, upsert=False,check_keys=False)
    c.close()    

def add_peer(peer, monitor, rtt):
    host=(socket.gethostname())
    c = MongoClient(host, 27017)
    db=c.tejo
    status=db.status
    monitors={monitor.split('.')[0]:rtt}
    status.insert({'peer':peer,'monitor':monitor,'target':monitor,'monitor_rtt':rtt,'target_rtt':rtt,'monitors':monitors,'active':False},check_keys=False)
    c.close()    

def remove_peer(peer):
    host=(socket.gethostname())
    c = MongoClient(host, 27017)
    db=c.tejo
    status=db.status
    status.remove({'peer':peer})
    c.close()    

def save_workload_list(workload_list,output_file='/home/user/workload_list.pck'):
    dic_node_file=output_file
    f = open(dic_node_file,'w')
    pickle.dump(workload_list, f)
    f.close()

def save_object_to_file(myobject,output_file):
    f = open(output_file,'w')
    pickle.dump(myobject, f)
    f.close()

def load_object_from_file(input_file):
    return pickle.load( open( input_file, "rb" ) )


if __name__ == '__main__':
    print "[%s]:update membership..."%(str(datetime.now()))    
    tejo_config=ConfigObj(TEJO_CONF_FILE)
    path_to_yanoama=tejo_config['root_dir']+'/yanoama'
    setup_peers_status_file=tejo_config['root_dir']+'/peers_tatus_table.pck'
    setup_peers_status={}
    nearest_peers_table={}
    if os.path.isfile(tejo_config['nearest_peers_file']):
        nearest_peers_table=load_object_from_file(tejo_config['nearest_peers_file'])
    if os.path.isfile(setup_peers_status_file):
        setup_peers_status=load_object_from_file(setup_peers_status_file)
        #cleanup list
        for peer in setup_peers_status:
            if not peer in nearest_peers_table:
                del setup_peers_status[peer] 
    else:
        for peer in nearest_peers_table:
            setup_peers_status[peer]={'rtt':nearest_peers_table[peer],'active':False}
     
        
    #getting non setuped nodes
    candidates=[]
    for peer in setup_peers_status:
        if not setup_peers_status[peer]['active']:
            candidates.append(peer)
            
    #to setup
    
    peer_to_setup=candidates[random.randrange(0,len(candidates))]
        
    script_path=tejo_config['home_dir']+'/contrib/pl/setup.sh'
    script_output = subprocess.Popen(['/bin/sh',script_path,peer_to_setup], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    script_path=tejo_config['home_dir']+'/tejo/common/experiments_scripts/peers/check_running_peer.sh'
    script_output = subprocess.Popen(['/bin/sh',script_path], stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip()
    if script_output=='ok':
        print '%s: ok!' % peer_to_setup
        setup_peers_status[peer_to_setup]['active']=True
        save_object_to_file(setup_peers_status, setup_peers_status_file)
    else:
        print '%s: failed!' % peer_to_setup



    print "[%s]:done."%(str(datetime.now()))  
    sys.exit(0)
        
#         
# #    save_workload_list(worload_peers, tejo_config['workload_list_file'])
#         
#     #2) get the list of ple nodes
#     
#     ###get monitor list
#     
#     
#     max_updates=10
#     steps=max_updates
#     
#     peers_to_remove=[]
#     while max_updates>0 and len(ple_nodes)>0:
#         node=ple_nodes[random.randrange(0,len(ple_nodes))]
#         ple_nodes.remove(node)
#         
#         rtt=getRTT_SSH(node,path_to_yanoama)
#         if rtt > 0.0:<commit_msg>replay<commit_after>import pickle,sys,socket,subprocess,os,git,random
from pymongo import MongoClient
from datetime import datetime
from time import mktime,sleep
#from planetlab import Monitor
from configobj import ConfigObj


try:
    import json
except ImportError:
    import simplejson as json

PLE_CONF_FILE='/etc/ple.conf'
TEJO_CONF_FILE='/etc/tejo.conf'



def clone_git_repository(uri_repository):
    subprocess.Popen(['git','clone',uri_repository], stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip()

def getRTT_SSH(hostname, yanoama_root):
    try:
        script_to_run=yanoama_root+'/yanoama/monitoring/get_rtt_ssh.sh'
        rtt=float(subprocess.Popen(['sh',script_to_run,hostname], stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip())
        if rtt>0.0:
            return rtt
        else:
            return -1
    except:
        return -1

#this returns a list of peers [ {k1:e1,...},...]
#each entry the the following structure:
#{u'rtt': 11.0, u'_id': ObjectId('55b955cf465cba3f890fa36e'), 
#  u'server': u'us-monitor-001.cloudapp.net', 
#  u'client': u'planetlab1.cs.ucla.edu'}
def get_list_of_peers():
    host=(socket.gethostname())
    c = MongoClient(host, 27017)
    db=c.tejo
    status=db.status
    list_of_peers=list(status.find())
    c.close()    
    return list_of_peers

def update_peer_monitor(peer, monitor, rtt,monitors):
    host=(socket.gethostname())
    c = MongoClient(host, 27017)
    db=c.tejo
    status=db.status
    status.update({'peer':peer},{'$set':{'monitor':monitor}}, upsert=False,check_keys=False)
    status.update({'peer':peer},{'$set':{'monitor_rtt':rtt}}, upsert=False,check_keys=False)
    status.update({'peer':peer},{'$set':{'monitors':monitors}}, upsert=False,check_keys=False)
    c.close()    

def add_peer(peer, monitor, rtt):
    host=(socket.gethostname())
    c = MongoClient(host, 27017)
    db=c.tejo
    status=db.status
    monitors={monitor.split('.')[0]:rtt}
    status.insert({'peer':peer,'monitor':monitor,'target':monitor,'monitor_rtt':rtt,'target_rtt':rtt,'monitors':monitors,'active':False},check_keys=False)
    c.close()    

def remove_peer(peer):
    host=(socket.gethostname())
    c = MongoClient(host, 27017)
    db=c.tejo
    status=db.status
    status.remove({'peer':peer})
    c.close()    

def save_workload_list(workload_list,output_file='/home/user/workload_list.pck'):
    dic_node_file=output_file
    f = open(dic_node_file,'w')
    pickle.dump(workload_list, f)
    f.close()

def save_object_to_file(myobject,output_file):
    f = open(output_file,'w')
    pickle.dump(myobject, f)
    f.close()

def load_object_from_file(input_file):
    return pickle.load( open( input_file, "rb" ) )


if __name__ == '__main__':
    print "[%s]:update membership..."%(str(datetime.now()))    
    tejo_config=ConfigObj(TEJO_CONF_FILE)
    path_to_yanoama=tejo_config['root_dir']+'/yanoama'
    setup_peers_status_file=tejo_config['root_dir']+'/peers_tatus_table.pck'
    setup_peers_status={}
    nearest_peers_table={}
    if os.path.isfile(tejo_config['nearest_peers_file']):
        nearest_peers_table=load_object_from_file(tejo_config['nearest_peers_file'])
    if os.path.isfile(setup_peers_status_file):
        setup_peers_status=load_object_from_file(setup_peers_status_file)
        #cleanup list
        for peer in setup_peers_status:
            if not peer in nearest_peers_table:
                del setup_peers_status[peer] 
    else:
        for peer in nearest_peers_table:
            setup_peers_status[peer]={'rtt':nearest_peers_table[peer],'active':False}
     
        
    #getting non setuped nodes
    candidates=[]
    for peer in setup_peers_status:
        if not setup_peers_status[peer]['active']:
            candidates.append(peer)
            
    #to setup
    
    peer_to_setup=candidates[random.randrange(0,len(candidates))]
        
    script_path=tejo_config['home_dir']+'/contrib/pl/setup.sh'
    script_output = subprocess.Popen(['/bin/sh',script_path,peer_to_setup], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    script_path=tejo_config['home_dir']+'/tejo/common/experiments_scripts/peers/check_running_peer.sh'
    key=tejo_config['root_dir']+"/.ssh/id_rsa_cloud"
    ssh_args=tejo_config['workload_user']+"@"+peer_to_setup    
    script_output = subprocess.Popen(['ssh','-i', key, "-o", "StrictHostKeyChecking=no", "-t", ssh_args,'/bin/sh',script_path], stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip()
    if script_output=='ok':
        print '%s: ok!' % peer_to_setup
        setup_peers_status[peer_to_setup]['active']=True
        save_object_to_file(setup_peers_status, setup_peers_status_file)
    else:
        print '%s: failed!' % peer_to_setup



    print "[%s]:done."%(str(datetime.now()))  
    sys.exit(0)
        
#         
# #    save_workload_list(worload_peers, tejo_config['workload_list_file'])
#         
#     #2) get the list of ple nodes
#     
#     ###get monitor list
#     
#     
#     max_updates=10
#     steps=max_updates
#     
#     peers_to_remove=[]
#     while max_updates>0 and len(ple_nodes)>0:
#         node=ple_nodes[random.randrange(0,len(ple_nodes))]
#         ple_nodes.remove(node)
#         
#         rtt=getRTT_SSH(node,path_to_yanoama)
#         if rtt > 0.0:
{'fraction_non_alphanumeric': 0.14140639969342786, 'fraction_numerical': 0.011879670434949224, 'mean_word_length': 3.914783427495292, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2363739', 'n_tokens_mistral': 3856, 'n_tokens_neox': 3779, 'n_words': 516}