<commit_before># Copyright 2011 Gilt Groupe, INC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
   The mothership module encompasses many of the operations that
   would need to be done against the mothership DB that cannot
   be easily encapsulated within the sqlalchemy models
"""
# imports
import mothership.network_mapper
import os
import re
import sys
import types

import mothership.kv
import mothership.list_values
import mothership.users
import mothership.validate

from mothership.mothership_models import *
from sqlalchemy import or_, desc, MetaData


# Mothership's main exception type
class MothershipError(Exception):
    pass


# Add a tag
def add_tag(cfg, name, start_port=None, stop_port=None, security_level=None):
    badtag = cfg.dbsess.query(Tag).\
              filter(Tag.name==name).first()

    if badtag:
        raise MothershipError("tag \"%s\" already exists, aborting" % name)

    r = Tag(name, start_port, stop_port, security_level)
    cfg.dbsess.add(r)
    cfg.dbsess.commit()


# Remove a tag
def rm_tag(cfg, name):
    r = cfg.dbsess.query(Tag).\
        filter(Tag.name==name).first()

    if not r:
        raise MothershipError("tag \"%s\" not found, aborting" % name)

    # modify all servers entries with pending delete tag
    for s in cfg.dbsess.query(Server).\
        filter(Server.tag==name):
        unqdn = '.'.join([s.hostname, s.realm, s.site_id])
        print '%s has primary tag: %s' % (unqdn, s.tag)
        ans = raw_input("specify replacement primary tag for %s: " % unqdn)
        mothership.modify_server_column(cfg, unqdn, 'tag', ans, True)
    # remove all kv entries with pending delete tag
    kvs = mothership.kv.collect(cfg, fqdn=None, key='tag', value=name)
    if kvs:
        print 'The following hosts have tag=%s:' % name
        for k in kvs:
            unqdn = '.'.join([k.hostname, k.realm, k.site_id])
            print '\t%s' % unqdn
    ans = raw_input("to delete tag \"%s\" please type \"delete_%s\": " % (name, name))
    if ans != "delete_%s" % name:
        raise MothershipError("aborted by user")
    else:
        for k in kvs:
            unqdn = '.'.join([k.hostname, k.realm, k.site_id])
            print 'Removing tag=%s from %s' % (name, unqdn)
            mothership.kv.delete(cfg, unqdn, key='tag', value=name)
        print 'Deleting tag: %s' % name
        cfg.dbsess.delete(r)
        cfg.dbsess.commit()


# display a tag
def display_tag(cfg, name):
    r = cfg.dbsess.query(Tag).\
        filter(Tag.name==name).first()

    if not r:
        raise MothershipError("tag \"%s\" not found, aborting" % name)
    else:
        print "name: %s\nstart_port: %s\nstop_port: %s\nsecurity level: %s" % (r.name, r.start_port, r.stop_port, r.security_level)


def calculate_last_virtual_ipaddress(cfg, vlan):
    data = cfg.dbsess.query(Network).\
        filter(Network.vlan==vlan).\
        filter(Network.hw_tag==cfg.dbnull)
    if data.first():
        data = data.order_by(Network.ip.desc()).first()
        return data.ip, data.realm, data.site_id
    else:
        # Assuming that all prod virtuals start on 1st_dyn_ip 
        ip,site_id = mothership.network_mapper.remap(cfg, ['1st_dyn_ip','siteid'], vlan=int(vlan))
        return ip, 'prod', site_id

def calculate_next_baremetal_vlan_ipaddress(cfg, vlan):
    data = cfg.dbsess.query(Network).\
        filter(Network.vlan==vlan).\
        filter(Network.ip!=cfg.dbnull).\
        filter(Network.hw_tag!=cfg.dbnull)
    lowest = mothership.network_mapper.remap(cfg, '1st_static_ip', vlan=int(vlan))
    try:
        first = data.order_by(Network.ip).first().ip
        last = data.order_by(Network.ip.desc()).first().ip
        #print 'RANGE: %s - %s' % (first, last)
    except:
        first = lowest
        last = first
    if first < lowest:
        first = lowest

    # compare the sequential list to the shiplist
    shiplist = []
    for i in data.order_by(Network.ip).all():
        shiplist.append(i.ip)

    # determine the next free ip and return it
    nextip = None
    for i in mothership.network_mapper.generate_ipaddress_list(first, last=last):
        if i not in shiplist:
            pingcheck = os.popen("ping -q -c2 -t2 "+i,"r")
            while 1:
                line = pingcheck.readline()
                if not line: break
                match = re.findall(re.compile(r"(\d)( packets)? received"),line)
                if match:
                    if int(match[0][0]) == 0:
                        return i
    if not nextip:
        nextip = mothership.network_mapper.next_ip(last)
    return nextip

def clear_serverinfo_from_network(cfg, server_id):
    for row in retrieve_network_rows(cfg, serverid=server_id):
        intf = getattr(row, 'interface')
        if intf not in ['drac', 'eth0']:
            for col in ['vlan', 'ip', 'netmask',
                'bond_options', 'public_ip', 'static_route']:
                setattr(row, col, cfg.dbnull)
        setattr(row, 'server_id', cfg.dbnull)
    cfg.dbsess.commit()

def confirm_column_change(curr_val, new_val, colname, tblname):
    print 'Please confirm the following change to:\n'
    print '%-10s | %-20s' % ('Table', 'Column')
    print '=' * 60
    print '%-10s | %-20s' % (tblname, colname)
    print ''
    print '%-20s --> %-20s' % (curr_val, new_val)
    ans = raw_input('\nTo confirm this change, type "modify_%s": ' % colname)
    if ans != 'modify_%s' % colname:
        print 'Modification of "%s" column in %s aborted.' % (colname, tblname)
        return False
    else:
        return True

def convert_drac_dict_to_network(cfg, drac_sysinfo, ip):
    dracip, siteid = mothership.network_mapper.remap(cfg,
        ['ip','siteid'], nic='drac', ip=ip)
    mgmtip = mothership.network_mapper.remap(cfg,
        'ip', nic='eth0', siteid=siteid)
    for k in sorted(drac_sysinfo.keys()):
        netdrac_sysinfo = drac_sysinfo.copy()
        netdrac_sysinfo.update({'realm':None,
            'interface':k, 'mac':netdrac_sysinfo[k]})
        if not k.startswith('eth') and k != 'drac': continue
        print 'Importing %s' % k
        net = mothership.network_mapper.remap(cfg,
            ['vlan','mask','ip', 'dom'], nic=k, siteid=siteid)
        if net:
            realm = mothership.validate.v_split_fqn(cfg, net[3])[1]
            if k!='eth1': # due to multiple vlans
                if net[2] == mgmtip:
                    netdrac_sysinfo.update({'ip':ip.replace(dracip,mgmtip)})
                elif net[2] == dracip:
                    netdrac_sysinfo.update({'ip':ip})
            netdrac_sysinfo.update({'vlan':net[0],
                'netmask':net[1], 'realm':realm})
        update_table_network(cfg, netdrac_sysinfo)

def convert_table_objects_to_dict(tables):
    d = {}
    for table in tables.keys():
        for k in getattr(tables, table).__dict__.keys():
            if k.startswith('_') or k=='id':
                continue
            d[k] = getattr(tables,table).__dict__[k]
    return d

def count_graveyard_server_by_date(cfg, hostname, when):
    host,realm,site_id = mothership.get_unqdn(cfg, hostname)
    try:<commit_msg>INFRAGEN-4835: passing a list instead of a string<commit_after># Copyright 2011 <NAME>, INC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
   The mothership module encompasses many of the operations that
   would need to be done against the mothership DB that cannot
   be easily encapsulated within the sqlalchemy models
"""
# imports
import mothership.network_mapper
import os
import re
import sys
import types

import mothership.kv
import mothership.list_values
import mothership.users
import mothership.validate

from mothership.mothership_models import *
from sqlalchemy import or_, desc, MetaData


# Mothership's main exception type
class MothershipError(Exception):
    pass


# Add a tag
def add_tag(cfg, name, start_port=None, stop_port=None, security_level=None):
    badtag = cfg.dbsess.query(Tag).\
              filter(Tag.name==name).first()

    if badtag:
        raise MothershipError("tag \"%s\" already exists, aborting" % name)

    r = Tag(name, start_port, stop_port, security_level)
    cfg.dbsess.add(r)
    cfg.dbsess.commit()


# Remove a tag
def rm_tag(cfg, name):
    r = cfg.dbsess.query(Tag).\
        filter(Tag.name==name).first()

    if not r:
        raise MothershipError("tag \"%s\" not found, aborting" % name)

    # modify all servers entries with pending delete tag
    for s in cfg.dbsess.query(Server).\
        filter(Server.tag==name):
        unqdn = '.'.join([s.hostname, s.realm, s.site_id])
        print '%s has primary tag: %s' % (unqdn, s.tag)
        ans = raw_input("specify replacement primary tag for %s: " % unqdn)
        mothership.modify_server_column(cfg, unqdn, 'tag', ans, True)
    # remove all kv entries with pending delete tag
    kvs = mothership.kv.collect(cfg, fqdn=None, key='tag', value=name)
    if kvs:
        print 'The following hosts have tag=%s:' % name
        for k in kvs:
            unqdn = '.'.join([k.hostname, k.realm, k.site_id])
            print '\t%s' % unqdn
    ans = raw_input("to delete tag \"%s\" please type \"delete_%s\": " % (name, name))
    if ans != "delete_%s" % name:
        raise MothershipError("aborted by user")
    else:
        for k in kvs:
            unqdn = '.'.join([k.hostname, k.realm, k.site_id])
            print 'Removing tag=%s from %s' % (name, unqdn)
            mothership.kv.delete(cfg, unqdn, key='tag', value=name)
        print 'Deleting tag: %s' % name
        cfg.dbsess.delete(r)
        cfg.dbsess.commit()


# display a tag
def display_tag(cfg, name):
    r = cfg.dbsess.query(Tag).\
        filter(Tag.name==name).first()

    if not r:
        raise MothershipError("tag \"%s\" not found, aborting" % name)
    else:
        print "name: %s\nstart_port: %s\nstop_port: %s\nsecurity level: %s" % (r.name, r.start_port, r.stop_port, r.security_level)


def calculate_last_virtual_ipaddress(cfg, vlan):
    data = cfg.dbsess.query(Network).\
        filter(Network.vlan==vlan).\
        filter(Network.hw_tag==cfg.dbnull)
    if data.first():
        data = data.order_by(Network.ip.desc()).first()
        return data.ip, data.realm, data.site_id
    else:
        # Assuming that all prod virtuals start on 1st_dyn_ip 
        ip,site_id = mothership.network_mapper.remap(cfg, ['1st_dyn_ip','siteid'], vlan=int(vlan))
        return ip, 'prod', site_id

def calculate_next_baremetal_vlan_ipaddress(cfg, vlan):
    data = cfg.dbsess.query(Network).\
        filter(Network.vlan==vlan).\
        filter(Network.ip!=cfg.dbnull).\
        filter(Network.hw_tag!=cfg.dbnull)
    lowest = mothership.network_mapper.remap(cfg, '1st_static_ip', vlan=int(vlan))
    try:
        first = data.order_by(Network.ip).first().ip
        last = data.order_by(Network.ip.desc()).first().ip
        #print 'RANGE: %s - %s' % (first, last)
    except:
        first = lowest
        last = first
    if first < lowest:
        first = lowest

    # compare the sequential list to the shiplist
    shiplist = []
    for i in data.order_by(Network.ip).all():
        shiplist.append(i.ip)

    # determine the next free ip and return it
    nextip = None
    for i in mothership.network_mapper.generate_ipaddress_list(first, last=last):
        if i not in shiplist:
            pingcheck = os.popen("ping -q -c2 -t2 "+i,"r")
            while 1:
                line = pingcheck.readline()
                if not line: break
                match = re.findall(re.compile(r"(\d)( packets)? received"),line)
                if match:
                    if int(match[0][0]) == 0:
                        return i
    if not nextip:
        nextip = mothership.network_mapper.next_ip(last)
    return nextip

def clear_serverinfo_from_network(cfg, server_id):
    for row in retrieve_network_rows(cfg, serverid=server_id):
        intf = getattr(row, 'interface')
        if intf not in ['drac', 'eth0']:
            for col in ['vlan', 'ip', 'netmask',
                'bond_options', 'public_ip', 'static_route']:
                setattr(row, col, cfg.dbnull)
        setattr(row, 'server_id', cfg.dbnull)
    cfg.dbsess.commit()

def confirm_column_change(curr_val, new_val, colname, tblname):
    print 'Please confirm the following change to:\n'
    print '%-10s | %-20s' % ('Table', 'Column')
    print '=' * 60
    print '%-10s | %-20s' % (tblname, colname)
    print ''
    print '%-20s --> %-20s' % (curr_val, new_val)
    ans = raw_input('\nTo confirm this change, type "modify_%s": ' % colname)
    if ans != 'modify_%s' % colname:
        print 'Modification of "%s" column in %s aborted.' % (colname, tblname)
        return False
    else:
        return True

def convert_drac_dict_to_network(cfg, drac_sysinfo, ip):
    dracip, siteid = mothership.network_mapper.remap(cfg,
        ['ip','siteid'], nic='drac', ip=ip)
    mgmtip = mothership.network_mapper.remap(cfg,
        'ip', nic='eth0', siteid=siteid)
    for k in sorted(drac_sysinfo.keys()):
        netdrac_sysinfo = drac_sysinfo.copy()
        netdrac_sysinfo.update({'realm':None,
            'interface':k, 'mac':netdrac_sysinfo[k]})
        if not k.startswith('eth') and k != 'drac': continue
        print 'Importing %s' % k
        net = mothership.network_mapper.remap(cfg,
            ['vlan','mask','ip', 'dom'], nic=k, siteid=siteid)
        if net:
            realm = mothership.validate.v_split_fqn(cfg, net[3][0])[1]
            if k!='eth1': # due to multiple vlans
                if net[2] == mgmtip:
                    netdrac_sysinfo.update({'ip':ip.replace(dracip,mgmtip)})
                elif net[2] == dracip:
                    netdrac_sysinfo.update({'ip':ip})
            netdrac_sysinfo.update({'vlan':net[0],
                'netmask':net[1], 'realm':realm})
        update_table_network(cfg, netdrac_sysinfo)

def convert_table_objects_to_dict(tables):
    d = {}
    for table in tables.keys():
        for k in getattr(tables, table).__dict__.keys():
            if k.startswith('_') or k=='id':
                continue
            d[k] = getattr(tables,table).__dict__[k]
    return d

def count_graveyard_server_by_date(cfg, hostname, when):
    host,realm,site_id = mothership.get_unqdn(cfg, hostname)
    try:
{'fraction_non_alphanumeric': 0.1281983991602152, 'fraction_numerical': 0.005576695971657263, 'mean_word_length': 2.705153135634419, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6777594', 'n_tokens_mistral': 5197, 'n_tokens_neox': 5004, 'n_words': 1315}