<commit_before>import urllib2
import json
import time
import threading
import sys

from sqlalchemy import exc

from config import panda_config
from pandalogger.PandaLogger import PandaLogger
import db_interface as dbif

_logger = PandaLogger().getLogger('configurator')
_session = dbif.get_session()


class Configurator(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self)

        if hasattr(panda_config,'AGIS_URL_SITES'):
            self.AGIS_URL_SITES = panda_config.AGIS_URL_SITES
        else:
            self.AGIS_URL_SITES = 'http://atlas-agis-api.cern.ch/request/site/query/?json&vo_name=atlas'
        self.site_dump = self.get_dump(self.AGIS_URL_SITES)

        if hasattr(panda_config,'AGIS_URL_DDMENDPOINTS'):
             self.AGIS_URL_DDMENDPOINTS = panda_config.AGIS_URL_DDMENDPOINTS
        else:
            self.AGIS_URL_DDMENDPOINTS = 'http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json'
        self.endpoint_dump = self.get_dump(self.AGIS_URL_DDMENDPOINTS)
        self.endpoint_token_dict = self.parse_endpoints()

        if hasattr(panda_config,'AGIS_URL_SCHEDCONFIG'):
             self.AGIS_URL_SCHEDCONFIG = panda_config.AGIS_URL_SCHEDCONFIG
        else:
            self.AGIS_URL_SCHEDCONFIG = 'http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json&preset=schedconf.all&vo_name=atlas'
        self.schedconfig_dump = self.get_dump(self.AGIS_URL_SCHEDCONFIG)
        

    def get_dump(self, url):
        response = urllib2.urlopen(url)
        json_str = response.read()
        dump = json.loads(json_str)
        return dump


    def get_site_info(self, site):
        """
        Gets the relevant information from a site
        """
        
        name = site['rc_site']
        
        #TODO: Think about the best way to store this information, also considering future requests
        if 'TaskNucleus' in site['datapolicies'] or site['rc_tier_level'] == 1:
            role = 'nucleus'
        else:
            role = 'satelite'
        
        state = site['rc_site_state']
        
        return (name, role, state)


    def parse_endpoints(self):
        """
        Puts the relevant information from endpoint_dump into a more usable format 
        """
        endpoint_token_dict = {}
        for endpoint in self.endpoint_dump:
            endpoint_token_dict[endpoint['name']] = {}
            endpoint_token_dict[endpoint['name']]['token'] = endpoint['token']
            endpoint_token_dict[endpoint['name']]['site_name'] = endpoint['rc_site']
        return endpoint_token_dict


    def process_dumps(self):
        """
        Parses the AGIS site and endpoint dumps 
        and prepares a format loadable to the DB
        """        
        #Variables that will contain only the relevant information
        sites_list = []
        included_sites = []
        ddm_endpoints_list = []
        panda_sites_list = []
        
        #Iterate the site dump
        for site in self.site_dump:
            #Add the site info to a list
            (site_name, site_role, site_state) = self.get_site_info(site)
            if site_name not in included_sites: #Avoid duplicate entries
                sites_list.append({'site_name': site_name, 'role': site_role, 'state': site_state})
                included_sites.append(site_name)
            
            #Get the DDM endpoints for the site we are inspecting
            for ddm_endpoint_name in site['ddmendpoints']:
                
                try:
                    ddm_spacetoken_name = self.endpoint_token_dict[ddm_endpoint_name]['token']
                except KeyError:
                    ddm_spacetoken_name = None
                    
                ddm_spacetoken_state = site['ddmendpoints'][ddm_endpoint_name]['state']
                
                ddm_endpoints_list.append({'ddm_endpoint_name': ddm_endpoint_name, 'site_name': site_name, 'ddm_spacetoken_name': ddm_spacetoken_name, 'state': ddm_spacetoken_state})

            #Get the PanDA resources 
            for panda_resource in site['presources']:
                for panda_site in site['presources'][panda_resource]:
                    panda_site_name = panda_site
                    panda_site_state = site['presources'][panda_resource][panda_site]['state']
                    panda_queue_name = None
                    for panda_queue in site['presources'][panda_resource][panda_site]['pandaqueues']:
                        panda_queue_name = panda_queue['name']
                    panda_sites_list.append({'panda_site_name': panda_site_name, 'panda_queue_name': panda_queue_name, 'site_name': site_name, 'state': panda_site_state})
        
        return sites_list, panda_sites_list, ddm_endpoints_list


    def process_schedconfig(self):
        """
        Gets PanDA site to DDM endpoint relationships from Schedconfig 
        and prepares a format loadable to the DB  
        """

        #relationship_tuples = dbif.read_panda_ddm_relationships_schedconfig(_session) #data almost as it comes from schedconfig
        relationships_list = [] #data to be loaded to configurator DB 
        
        for panda_site in self.schedconfig_dump:
            count = 0
            ddm_endpoints = [ddm_endpoint.strip() for ddm_endpoint in self.schedconfig_dump[panda_site]['ddm'].split(',')]
            for ddm_endpoint_name in ddm_endpoints:
                #The first DDM endpoint in the list should be the primary
                if count == 0:
                    is_default = 'Y'
                else:
                    is_default = 'N'
                
                #Check if the ddm_endpoint and the panda_site belong to the same site
                site_name_endpoint = self.endpoint_token_dict[ddm_endpoint_name]['site_name']
                site_name_pandasite = self.schedconfig_dump[panda_site]['rc_site']
                if site_name_endpoint == site_name_pandasite:
                    is_local = 'Y'
                else:
                    is_local = 'N'
                 
                relationships_list.append({'panda_site_name': panda_site_name, 
                                           'ddm_endpoint_name': ddm_endpoint_name,<commit_msg>Corrected variable names in process_schedconfig<commit_after>import urllib2
import json
import time
import threading
import sys

from sqlalchemy import exc

from config import panda_config
from pandalogger.PandaLogger import PandaLogger
import db_interface as dbif

_logger = PandaLogger().getLogger('configurator')
_session = dbif.get_session()


class Configurator(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self)

        if hasattr(panda_config,'AGIS_URL_SITES'):
            self.AGIS_URL_SITES = panda_config.AGIS_URL_SITES
        else:
            self.AGIS_URL_SITES = 'http://atlas-agis-api.cern.ch/request/site/query/?json&vo_name=atlas'
        _logger.debug('Getting site dump...')
        self.site_dump = self.get_dump(self.AGIS_URL_SITES)
        _logger.debug('...done')

        if hasattr(panda_config,'AGIS_URL_DDMENDPOINTS'):
             self.AGIS_URL_DDMENDPOINTS = panda_config.AGIS_URL_DDMENDPOINTS
        else:
            self.AGIS_URL_DDMENDPOINTS = 'http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json'
        _logger.debug('Getting DDM endpoints dump...')
        self.endpoint_dump = self.get_dump(self.AGIS_URL_DDMENDPOINTS)
        _logger.debug('...done')
        _logger.debug('Parsing endpoints...')
        self.endpoint_token_dict = self.parse_endpoints()
        _logger.debug('...done')

        if hasattr(panda_config,'AGIS_URL_SCHEDCONFIG'):
             self.AGIS_URL_SCHEDCONFIG = panda_config.AGIS_URL_SCHEDCONFIG
        else:
            self.AGIS_URL_SCHEDCONFIG = 'http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json&preset=schedconf.all&vo_name=atlas'
        _logger.debug('Getting schedconfig dump...')
        self.schedconfig_dump = self.get_dump(self.AGIS_URL_SCHEDCONFIG)
        _logger.debug('...done')
        

    def get_dump(self, url):
        response = urllib2.urlopen(url)
        json_str = response.read()
        dump = json.loads(json_str)
        return dump


    def get_site_info(self, site):
        """
        Gets the relevant information from a site
        """
        
        name = site['rc_site']
        
        #TODO: Think about the best way to store this information, also considering future requests
        if 'TaskNucleus' in site['datapolicies'] or site['rc_tier_level'] == 1:
            role = 'nucleus'
        else:
            role = 'satelite'
        
        state = site['rc_site_state']
        
        return (name, role, state)


    def parse_endpoints(self):
        """
        Puts the relevant information from endpoint_dump into a more usable format 
        """
        endpoint_token_dict = {}
        for endpoint in self.endpoint_dump:
            endpoint_token_dict[endpoint['name']] = {}
            endpoint_token_dict[endpoint['name']]['token'] = endpoint['token']
            endpoint_token_dict[endpoint['name']]['site_name'] = endpoint['rc_site']
        return endpoint_token_dict


    def process_dumps(self):
        """
        Parses the AGIS site and endpoint dumps 
        and prepares a format loadable to the DB
        """        
        #Variables that will contain only the relevant information
        sites_list = []
        included_sites = []
        ddm_endpoints_list = []
        panda_sites_list = []
        
        #Iterate the site dump
        for site in self.site_dump:
            #Add the site info to a list
            (site_name, site_role, site_state) = self.get_site_info(site)
            if site_name not in included_sites: #Avoid duplicate entries
                sites_list.append({'site_name': site_name, 'role': site_role, 'state': site_state})
                included_sites.append(site_name)
            
            #Get the DDM endpoints for the site we are inspecting
            for ddm_endpoint_name in site['ddmendpoints']:
                
                try:
                    ddm_spacetoken_name = self.endpoint_token_dict[ddm_endpoint_name]['token']
                except KeyError:
                    ddm_spacetoken_name = None
                    
                ddm_spacetoken_state = site['ddmendpoints'][ddm_endpoint_name]['state']
                
                ddm_endpoints_list.append({'ddm_endpoint_name': ddm_endpoint_name, 'site_name': site_name, 'ddm_spacetoken_name': ddm_spacetoken_name, 'state': ddm_spacetoken_state})

            #Get the PanDA resources 
            for panda_resource in site['presources']:
                for panda_site in site['presources'][panda_resource]:
                    panda_site_name = panda_site
                    panda_site_state = site['presources'][panda_resource][panda_site]['state']
                    panda_queue_name = None
                    for panda_queue in site['presources'][panda_resource][panda_site]['pandaqueues']:
                        panda_queue_name = panda_queue['name']
                    panda_sites_list.append({'panda_site_name': panda_site_name, 'panda_queue_name': panda_queue_name, 'site_name': site_name, 'state': panda_site_state})
        
        return sites_list, panda_sites_list, ddm_endpoints_list


    def process_schedconfig(self):
        """
        Gets PanDA site to DDM endpoint relationships from Schedconfig 
        and prepares a format loadable to the DB  
        """

        #relationship_tuples = dbif.read_panda_ddm_relationships_schedconfig(_session) #data almost as it comes from schedconfig
        relationships_list = [] #data to be loaded to configurator DB 
        
        for panda_site_name in self.schedconfig_dump:
            count = 0
            ddm_endpoints = [ddm_endpoint.strip() for ddm_endpoint in self.schedconfig_dump[panda_site_name]['ddm'].split(',')]
            for ddm_endpoint_name in ddm_endpoints:
                #The first DDM endpoint in the list should be the primary
                if count == 0:
                    is_default = 'Y'
                else:
                    is_default = 'N'
                
                #Check if the ddm_endpoint and the panda_site belong to the same site
                site_name_endpoint = self.endpoint_token_dict[ddm_endpoint_name]['site_name']
                site_name_pandasite = self.schedconfig_dump[panda_site_name]['rc_site']
                if site_name_endpoint == site_name_pandasite:
                    is_local = 'Y'
                else:
                    is_local = 'N'
                 
                relationships_list.append({'panda_site_name': panda_site_name, 
                                           'ddm_endpoint_name': ddm_endpoint_name,
{'fraction_non_alphanumeric': 0.08508983433149257, 'fraction_numerical': 0.0007777864198491094, 'mean_word_length': 2.221748935103984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7236190', 'n_tokens_mistral': 3898, 'n_tokens_neox': 3745, 'n_words': 765}