<commit_before>#!/usr/bin/python
from collections import defaultdict
from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
from sfa.util.sfatime import utcparse, datetime_to_string
from sfa.util.sfalogging import logger

from sfa.rspecs.rspec import RSpec
from sfa.rspecs.elements.hardware_type import HardwareType
from sfa.rspecs.elements.node import Node
from sfa.rspecs.elements.link import Link
from sfa.rspecs.elements.sliver import Sliver
from sfa.rspecs.elements.login import Login
from sfa.rspecs.elements.location import Location
from sfa.rspecs.elements.interface import Interface
from sfa.rspecs.elements.services import Services
from sfa.rspecs.elements.pltag import PLTag
from sfa.rspecs.elements.lease import Lease
from sfa.rspecs.elements.granularity import Granularity
from sfa.rspecs.version_manager import VersionManager

from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
from sfa.planetlab.vlink import get_tc_rate
from sfa.planetlab.topology import Topology

import time

class PlAggregate:

    def __init__(self, driver):
        self.driver = driver

    def get_nodes(self, options={}):
        filter = {'peer_id': None}
        geni_available = options.get('geni_available')    
        if geni_available == True:
            filter['boot_state'] = 'boot'
        nodes = self.driver.shell.GetNodes(filter)
       
        return nodes  
 
    def get_sites(self, filter={}):
        sites = {}
        for site in self.driver.shell.GetSites(filter):
            sites[site['site_id']] = site
        return sites

    def get_interfaces(self, filter={}):
        interfaces = {}
        for interface in self.driver.shell.GetInterfaces(filter):
            iface = Interface()
            if interface['bwlimit']:
                interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
            interfaces[interface['interface_id']] = interface
        return interfaces

    def get_links(self, sites, nodes, interfaces):
        
        topology = Topology() 
        links = []
        for (site_id1, site_id2) in topology:
            site_id1 = int(site_id1)
            site_id2 = int(site_id2)
            link = Link()
            if not site_id1 in sites or site_id2 not in sites:
                continue
            site1 = sites[site_id1]
            site2 = sites[site_id2]
            # get hrns
            site1_hrn = self.driver.hrn + '.' + site1['login_base']
            site2_hrn = self.driver.hrn + '.' + site2['login_base']

            for s1_node_id in site1['node_ids']:
                for s2_node_id in site2['node_ids']:
                    if s1_node_id not in nodes or s2_node_id not in nodes:
                        continue
                    node1 = nodes[s1_node_id]
                    node2 = nodes[s2_node_id]
                    # set interfaces
                    # just get first interface of the first node
                    if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
                    if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
                    if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
                    if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']

                    if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
                    if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )

                    # set link
                    link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
                    link['interface1'] = if1
                    link['interface2'] = if2
                    link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
                    link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
                    link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
                    links.append(link)

        return links

    def get_node_tags(self, filter={}):
        node_tags = {}
        for node_tag in self.driver.shell.GetNodeTags(filter):
            node_tags[node_tag['node_tag_id']] = node_tag
        return node_tags

    def get_pl_initscripts(self, filter={}):
        pl_initscripts = {}
        filter.update({'enabled': True})
        for initscript in self.driver.shell.GetInitScripts(filter):
            pl_initscripts[initscript['initscript_id']] = initscript
        return pl_initscripts

    def get_slivers(self, urns, options):
        names = set()
        ids = set()
        for urn in urns:
            xrn = PlXrn(xrn=urn)
            names.add(xrn.get_slice_name())
            if xrn.id:
                ids.add(xrn.id)

        slices = self.driver.shell.GetSlices(names)
        # filter on id
        if ids:
            slices = [slice for slice in slices if slice['slice_id'] in ids]

        tags_dict = self.get_slice_tags(slices)
        nodes_dict = self.get_slice_nodes(slices, options)
        slivers = []
        for node in nodes_dict.values():
            sliver = node.update(slices[0]) 
            sliver['tags'] = tags_dict[node['node_id']]
        return slivers

    def node_to_rspec_node(self, sites, interfaces, tags, pl_initscripts=[], grain=None, options={}):
        rspec_node = Node()
        # xxx how to retrieve site['login_base']
        site=sites_dict[node['site_id']]
        rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
        rspec_node['component_name'] = node['hostname']
        rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
        rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
        # do not include boot state (<available> element) in the manifest rspec
        rspec_node['boot_state'] = node['boot_state']
        rspec_node['exclusive'] = 'false'
        rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
                                        HardwareType({'name': 'pc'})]
        # only doing this because protogeni rspec needs
        # to advertise available initscripts
        rspec_node['pl_initscripts'] = pl_initscripts.values()
         # add site/interface info to nodes.
        # assumes that sites, interfaces and tags have already been prepared.
        if site['longitude'] and site['latitude']:
            location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
            rspec_node['location'] = location
        # Granularity
        granularity = Granularity({'grain': grain})
        rspec_node['granularity'] = granularity
        rspec_node['interfaces'] = []
        if_count=0
        for if_id in node['interface_ids']:
                interface = Interface(interfaces[if_id])
                interface['ipv4'] = interface['ip']
                interface['component_id'] = PlXrn(auth=self.driver.hrn,
                                                  interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
                # interfaces in the manifest need a client id
                if slice:
                    interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
                rspec_node['interfaces'].append(interface)
                if_count+=1

            tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']]
            rspec_node['tags'] = tags
        return rspec_node

    def sliver_to_rspec_node(self, sliver):
        # get the granularity in second for the reservation system
        grain = self.driver.shell.GetLeaseGranularity()
        if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
            continue<commit_msg>syntax error again (indentation)<commit_after>#!/usr/bin/python
from collections import defaultdict
from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn
from sfa.util.sfatime import utcparse, datetime_to_string
from sfa.util.sfalogging import logger

from sfa.rspecs.rspec import RSpec
from sfa.rspecs.elements.hardware_type import HardwareType
from sfa.rspecs.elements.node import Node
from sfa.rspecs.elements.link import Link
from sfa.rspecs.elements.sliver import Sliver
from sfa.rspecs.elements.login import Login
from sfa.rspecs.elements.location import Location
from sfa.rspecs.elements.interface import Interface
from sfa.rspecs.elements.services import Services
from sfa.rspecs.elements.pltag import PLTag
from sfa.rspecs.elements.lease import Lease
from sfa.rspecs.elements.granularity import Granularity
from sfa.rspecs.version_manager import VersionManager

from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn
from sfa.planetlab.vlink import get_tc_rate
from sfa.planetlab.topology import Topology

import time

class PlAggregate:

    def __init__(self, driver):
        self.driver = driver

    def get_nodes(self, options={}):
        filter = {'peer_id': None}
        geni_available = options.get('geni_available')    
        if geni_available == True:
            filter['boot_state'] = 'boot'
        nodes = self.driver.shell.GetNodes(filter)
       
        return nodes  
 
    def get_sites(self, filter={}):
        sites = {}
        for site in self.driver.shell.GetSites(filter):
            sites[site['site_id']] = site
        return sites

    def get_interfaces(self, filter={}):
        interfaces = {}
        for interface in self.driver.shell.GetInterfaces(filter):
            iface = Interface()
            if interface['bwlimit']:
                interface['bwlimit'] = str(int(interface['bwlimit'])/1000)
            interfaces[interface['interface_id']] = interface
        return interfaces

    def get_links(self, sites, nodes, interfaces):
        
        topology = Topology() 
        links = []
        for (site_id1, site_id2) in topology:
            site_id1 = int(site_id1)
            site_id2 = int(site_id2)
            link = Link()
            if not site_id1 in sites or site_id2 not in sites:
                continue
            site1 = sites[site_id1]
            site2 = sites[site_id2]
            # get hrns
            site1_hrn = self.driver.hrn + '.' + site1['login_base']
            site2_hrn = self.driver.hrn + '.' + site2['login_base']

            for s1_node_id in site1['node_ids']:
                for s2_node_id in site2['node_ids']:
                    if s1_node_id not in nodes or s2_node_id not in nodes:
                        continue
                    node1 = nodes[s1_node_id]
                    node2 = nodes[s2_node_id]
                    # set interfaces
                    # just get first interface of the first node
                    if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id']))
                    if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip']
                    if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id']))
                    if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip']

                    if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} )
                    if2 = Interface({'component_id': if2_xrn.urn, 'ipv4': if2_ipv4} )

                    # set link
                    link = Link({'capacity': '1000000', 'latency': '0', 'packet_loss': '0', 'type': 'ipv4'})
                    link['interface1'] = if1
                    link['interface2'] = if2
                    link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base'])
                    link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn()
                    link['component_manager_id'] =  hrn_to_urn(self.driver.hrn, 'authority+am')
                    links.append(link)

        return links

    def get_node_tags(self, filter={}):
        node_tags = {}
        for node_tag in self.driver.shell.GetNodeTags(filter):
            node_tags[node_tag['node_tag_id']] = node_tag
        return node_tags

    def get_pl_initscripts(self, filter={}):
        pl_initscripts = {}
        filter.update({'enabled': True})
        for initscript in self.driver.shell.GetInitScripts(filter):
            pl_initscripts[initscript['initscript_id']] = initscript
        return pl_initscripts

    def get_slivers(self, urns, options):
        names = set()
        ids = set()
        for urn in urns:
            xrn = PlXrn(xrn=urn)
            names.add(xrn.get_slice_name())
            if xrn.id:
                ids.add(xrn.id)

        slices = self.driver.shell.GetSlices(names)
        # filter on id
        if ids:
            slices = [slice for slice in slices if slice['slice_id'] in ids]

        tags_dict = self.get_slice_tags(slices)
        nodes_dict = self.get_slice_nodes(slices, options)
        slivers = []
        for node in nodes_dict.values():
            sliver = node.update(slices[0]) 
            sliver['tags'] = tags_dict[node['node_id']]
        return slivers

    def node_to_rspec_node(self, sites, interfaces, tags, pl_initscripts=[], grain=None, options={}):
        rspec_node = Node()
        # xxx how to retrieve site['login_base']
        site=sites_dict[node['site_id']]
        rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname'])
        rspec_node['component_name'] = node['hostname']
        rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
        rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa')
        # do not include boot state (<available> element) in the manifest rspec
        rspec_node['boot_state'] = node['boot_state']
        rspec_node['exclusive'] = 'false'
        rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}),
                                        HardwareType({'name': 'pc'})]
        # only doing this because protogeni rspec needs
        # to advertise available initscripts
        rspec_node['pl_initscripts'] = pl_initscripts.values()
         # add site/interface info to nodes.
        # assumes that sites, interfaces and tags have already been prepared.
        if site['longitude'] and site['latitude']:
            location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'})
            rspec_node['location'] = location
        # Granularity
        granularity = Granularity({'grain': grain})
        rspec_node['granularity'] = granularity
        rspec_node['interfaces'] = []
        if_count=0
        for if_id in node['interface_ids']:
            interface = Interface(interfaces[if_id])
            interface['ipv4'] = interface['ip']
            interface['component_id'] = PlXrn(auth=self.driver.hrn,
                                              interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn()
            # interfaces in the manifest need a client id
            if slice:
                interface['client_id'] = "%s:%s" % (node['node_id'], if_id)
            rspec_node['interfaces'].append(interface)
            if_count+=1

        tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']]
        rspec_node['tags'] = tags
        return rspec_node

    def sliver_to_rspec_node(self, sliver):
        # get the granularity in second for the reservation system
        grain = self.driver.shell.GetLeaseGranularity()
        if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']:
            continue
{'fraction_non_alphanumeric': 0.11465533641955736, 'fraction_numerical': 0.00938550320248589, 'mean_word_length': 2.4119428818693205, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7561524', 'n_tokens_mistral': 4961, 'n_tokens_neox': 4863, 'n_words': 1010}