<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Aws Provider Class
"""

from ipaddress import ip_network
from kvirt import common
from kvirt.common import pprint, error, warning, get_ssh_pub_key
from kvirt.defaults import METADATA_FIELDS
import boto3
import os
import sys
from socket import gethostbyname
from string import ascii_lowercase
from time import sleep
import webbrowser

staticf = {'t2.nano': {'cpus': 1, 'memory': 512}, 't2.micro': {'cpus': 1, 'memory': 1024},
           't2.small': {'cpus': 1, 'memory': 2048}, 't2.medium': {'cpus': 2, 'memory': 4096},
           't2.large': {'cpus': 2, 'memory': 8144}, 't2.xlarge': {'cpus': 2, 'memory': 16384},
           'm5.large': {'cpus': 2, 'memory': 8144}, 'm5.xlarge': {'cpus': 4, 'memory': 16384},
           'm5.2xlarge': {'cpus': 8, 'memory': 32768}, 'm5.4xlarge': {'cpus': 16, 'memory': 65536}
           }


class Kaws(object):
    """

    """
    def __init__(self, access_key_id=None, access_key_secret=None, debug=False,
                 region='eu-west-3', keypair=None, session_token=None):
        self.ami_date = 20195
        self.debug = debug
        self.conn = boto3.client('ec2', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                                 region_name=region, aws_session_token=session_token)
        self.resource = boto3.resource('ec2', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                                       region_name=region, aws_session_token=session_token)
        self.dns = boto3.client('route53', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                                region_name=region, aws_session_token=session_token)
        self.elb = boto3.client('elb', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                                region_name=region, aws_session_token=session_token)
        self.s3 = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                               region_name=region, aws_session_token=session_token)
        self.access_key_id = access_key_id
        self.access_key_secret = access_key_secret
        self.region = region
        self.keypair = keypair
        return

    def close(self):
        return

    def exists(self, name):
        conn = self.conn
        try:
            Filters = {'Name': "tag:Name", 'Values': [name]}
            conn.describe_instances(Filters=[Filters])['Reservations'][0]['Instances'][0]
            return True
        except:
            return False

    def net_exists(self, name):
        conn = self.conn
        try:
            conn.describe_subnets(SubnetIds=[name])
            return True
        except:
            return False

    def disk_exists(self, pool, name):
        print("not implemented")

    def create(self, name, virttype=None, profile='', flavor=None, plan='kvirt', cpumodel='Westmere', cpuflags=[],
               cpupinning=[], numcpus=2, memory=512, guestid='guestrhel764', pool='default', image=None,
               disks=[{'size': 10}], disksize=10, diskthin=True, diskinterface='virtio', nets=['default'], iso=None,
               vnc=True, cloudinit=True, reserveip=False, reservedns=False, reservehost=False, start=True, keys=[],
               cmds=[], ips=None, netmasks=None, gateway=None, nested=True, dns=None, domain=None, tunnel=False,
               files=[], enableroot=True, alias=[], overrides={}, tags=[], storemetadata=False,
               sharedfolders=[], kernel=None, initrd=None, cmdline=None, placement=[], autostart=False,
               cpuhotplug=False, memoryhotplug=False, numamode=None, numa=[], pcidevices=[], tpm=False, rng=False,
               metadata={}, securitygroups=[], vmuser=None):
        conn = self.conn
        if self.exists(name):
            return {'result': 'failure', 'reason': f"VM {name} already exists"}
        image = self.__evaluate_image(image)
        keypair = self.keypair
        if image is None:
            return {'result': 'failure', 'reason': 'An image (or amid) is required'}
        else:
            _filter = 'image-id' if image.startswith('ami-') else 'name'
            Filters = [{'Name': _filter, 'Values': [image]}]
            images = conn.describe_images(Filters=Filters)
            if 'Images' in images and images['Images']:
                imageinfo = images['Images'][0]
                imageid = imageinfo['ImageId']
                if _filter == 'name':
                    pprint(f"Using ami {imageid}")
                image = imageinfo['Name']
            else:
                return {'result': 'failure', 'reason': f'Invalid image {image}'}
        defaultsubnetid = None
        if flavor is None:
            matching = [f for f in staticf if staticf[f]['cpus'] >= numcpus and staticf[f]['memory'] >= memory]
            if matching:
                flavor = matching[0]
                pprint(f"Using instance type {flavor}")
            else:
                return {'result': 'failure', 'reason': 'Couldnt find instance type matching requirements'}
        vmtags = [{'ResourceType': 'instance',
                   'Tags': [{'Key': 'Name', 'Value': name}, {'Key': 'hostname', 'Value': name}]}]
        for entry in [field for field in metadata if field in METADATA_FIELDS]:
            vmtags[0]['Tags'].append({'Key': entry, 'Value': metadata[entry]})
        if keypair is None:
            keypair = f'kvirt_{self.access_key_id}'
        keypairs = [k for k in conn.describe_key_pairs()['KeyPairs'] if k['KeyName'] == keypair]
        if not keypairs:
            pprint(f"Importing your public key as {keypair}")
            publickeyfile = get_ssh_pub_key()
            if publickeyfile is None:
                error("No public key found. Leaving")
                return {'result': 'failure', 'reason': 'No public key found'}
            publickeyfile = open(publickeyfile).read()
            conn.import_key_pair(KeyName=keypair, PublicKeyMaterial=publickeyfile)
        if cloudinit:
            if image is not None and common.needs_ignition(image):
                version = common.ignition_version(image)
                userdata = common.ignition(name=name, keys=keys, cmds=cmds, nets=nets, gateway=gateway, dns=dns,
                                           domain=domain, files=files, enableroot=enableroot,
                                           overrides=overrides, version=version, plan=plan, image=image,
                                           vmuser=vmuser)
            else:
                userdata = common.cloudinit(name=name, keys=keys, cmds=cmds, nets=nets, gateway=gateway, dns=dns,
                                            domain=domain, files=files, enableroot=enableroot,
                                            overrides=overrides, fqdn=True, storemetadata=storemetadata,
                                            vmuser=vmuser)[0]
        else:
            userdata = ''
        networkinterfaces = []
        blockdevicemappings = []
        privateips = []
        vpcs = conn.describe_vpcs()
        subnets = conn.describe_subnets()
        for index, net in enumerate(nets):
            networkinterface = {'DeleteOnTermination': True, 'Description': f"eth{index}", 'DeviceIndex': index,
                                'Groups': ['string'], 'SubnetId': 'string'}
            ip = None
            if isinstance(net, str):
                netname = net
                netpublic = True
            elif isinstance(net, dict) and 'name' in net:
                netname = net['name']
                ip = net.get('ip')
                alias = net.get('alias')
                netpublic = net.get('public', True)
            # if securitygroups:
            #    netpublic = False
            networkinterface['AssociatePublicIpAddress'] = netpublic if index == 0 else False
            if netname in [subnet['SubnetId'] for subnet in subnets['Subnets']]:
                vpcid = [subnet['VpcId'] for subnet in subnets['Subnets'] if subnet['SubnetId'] == netname][0]
            elif netname == 'default':
                if defaultsubnetid is not None:
                    netname = defaultsubnetid
                else:
                    vpcid = [vpc['VpcId'] for vpc in vpcs['Vpcs'] if vpc['IsDefault']][0]
                    subnetid = [subnet['SubnetId'] for subnet in subnets['Subnets']
                                if subnet['DefaultForAz'] and subnet['VpcId'] == vpcid][0]
                    netname = subnetid
                    defaultsubnetid = netname
                    pprint(f"Using subnet {defaultsubnetid} as default")
            else:
                vpcid = self.get_vpc_id(vpcs, netname) if not netname.startswith('vpc-') else netname
                if vpcid is None:
                    error(f"Couldn't find vpc {netname}")
                    sys.exit(1)
                subnetids = [subnet['SubnetId'] for subnet in subnets['Subnets'] if subnet['VpcId'] == vpcid]
                if subnetids:
                    netname = subnetids[0]
                else:
                    error(f"Couldn't find valid subnet for vpc {netname}")
                    sys.exit(1)
            if ips and len(ips) > index and ips[index] is not None:
                ip = ips[index]
                if index == 0:
                    networkinterface['PrivateIpAddress'] = ip
                    privateip = {'Primary': True, 'PrivateIpAddress': ip}
                else:
                    privateip = {'Primary': False, 'PrivateIpAddress': ip}
                privateips.append(privateip)
            networkinterface['SubnetId'] = netname
            if index == 0:
                SecurityGroupIds = []
                for sg in securitygroups:
                    sgid = self.get_security_group_id(sg, vpcid)
                    if sgid is not None:
                        SecurityGroupIds.append(sgid)
                if 'kubetype' in metadata and metadata['kubetype'] == "openshift":
                    kube = metadata['kube']
                    pprint(f"Adding vm to security group {kube}")
                    kubesgid = self.get_security_group_id(kube, vpcid)
                    if kubesgid is None:
                        sg = self.resource.create_security_group(GroupId=kubesgid, Description=kube, VpcId=vpcid)<commit_msg>aws: fix create security group typo<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Aws Provider Class
"""

from ipaddress import ip_network
from kvirt import common
from kvirt.common import pprint, error, warning, get_ssh_pub_key
from kvirt.defaults import METADATA_FIELDS
import boto3
import os
import sys
from socket import gethostbyname
from string import ascii_lowercase
from time import sleep
import webbrowser

staticf = {'t2.nano': {'cpus': 1, 'memory': 512}, 't2.micro': {'cpus': 1, 'memory': 1024},
           't2.small': {'cpus': 1, 'memory': 2048}, 't2.medium': {'cpus': 2, 'memory': 4096},
           't2.large': {'cpus': 2, 'memory': 8144}, 't2.xlarge': {'cpus': 2, 'memory': 16384},
           'm5.large': {'cpus': 2, 'memory': 8144}, 'm5.xlarge': {'cpus': 4, 'memory': 16384},
           'm5.2xlarge': {'cpus': 8, 'memory': 32768}, 'm5.4xlarge': {'cpus': 16, 'memory': 65536}
           }


class Kaws(object):
    """

    """
    def __init__(self, access_key_id=None, access_key_secret=None, debug=False,
                 region='eu-west-3', keypair=None, session_token=None):
        self.ami_date = 20195
        self.debug = debug
        self.conn = boto3.client('ec2', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                                 region_name=region, aws_session_token=session_token)
        self.resource = boto3.resource('ec2', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                                       region_name=region, aws_session_token=session_token)
        self.dns = boto3.client('route53', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                                region_name=region, aws_session_token=session_token)
        self.elb = boto3.client('elb', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                                region_name=region, aws_session_token=session_token)
        self.s3 = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret,
                               region_name=region, aws_session_token=session_token)
        self.access_key_id = access_key_id
        self.access_key_secret = access_key_secret
        self.region = region
        self.keypair = keypair
        return

    def close(self):
        return

    def exists(self, name):
        conn = self.conn
        try:
            Filters = {'Name': "tag:Name", 'Values': [name]}
            conn.describe_instances(Filters=[Filters])['Reservations'][0]['Instances'][0]
            return True
        except:
            return False

    def net_exists(self, name):
        conn = self.conn
        try:
            conn.describe_subnets(SubnetIds=[name])
            return True
        except:
            return False

    def disk_exists(self, pool, name):
        print("not implemented")

    def create(self, name, virttype=None, profile='', flavor=None, plan='kvirt', cpumodel='Westmere', cpuflags=[],
               cpupinning=[], numcpus=2, memory=512, guestid='guestrhel764', pool='default', image=None,
               disks=[{'size': 10}], disksize=10, diskthin=True, diskinterface='virtio', nets=['default'], iso=None,
               vnc=True, cloudinit=True, reserveip=False, reservedns=False, reservehost=False, start=True, keys=[],
               cmds=[], ips=None, netmasks=None, gateway=None, nested=True, dns=None, domain=None, tunnel=False,
               files=[], enableroot=True, alias=[], overrides={}, tags=[], storemetadata=False,
               sharedfolders=[], kernel=None, initrd=None, cmdline=None, placement=[], autostart=False,
               cpuhotplug=False, memoryhotplug=False, numamode=None, numa=[], pcidevices=[], tpm=False, rng=False,
               metadata={}, securitygroups=[], vmuser=None):
        conn = self.conn
        if self.exists(name):
            return {'result': 'failure', 'reason': f"VM {name} already exists"}
        image = self.__evaluate_image(image)
        keypair = self.keypair
        if image is None:
            return {'result': 'failure', 'reason': 'An image (or amid) is required'}
        else:
            _filter = 'image-id' if image.startswith('ami-') else 'name'
            Filters = [{'Name': _filter, 'Values': [image]}]
            images = conn.describe_images(Filters=Filters)
            if 'Images' in images and images['Images']:
                imageinfo = images['Images'][0]
                imageid = imageinfo['ImageId']
                if _filter == 'name':
                    pprint(f"Using ami {imageid}")
                image = imageinfo['Name']
            else:
                return {'result': 'failure', 'reason': f'Invalid image {image}'}
        defaultsubnetid = None
        if flavor is None:
            matching = [f for f in staticf if staticf[f]['cpus'] >= numcpus and staticf[f]['memory'] >= memory]
            if matching:
                flavor = matching[0]
                pprint(f"Using instance type {flavor}")
            else:
                return {'result': 'failure', 'reason': 'Couldnt find instance type matching requirements'}
        vmtags = [{'ResourceType': 'instance',
                   'Tags': [{'Key': 'Name', 'Value': name}, {'Key': 'hostname', 'Value': name}]}]
        for entry in [field for field in metadata if field in METADATA_FIELDS]:
            vmtags[0]['Tags'].append({'Key': entry, 'Value': metadata[entry]})
        if keypair is None:
            keypair = f'kvirt_{self.access_key_id}'
        keypairs = [k for k in conn.describe_key_pairs()['KeyPairs'] if k['KeyName'] == keypair]
        if not keypairs:
            pprint(f"Importing your public key as {keypair}")
            publickeyfile = get_ssh_pub_key()
            if publickeyfile is None:
                error("No public key found. Leaving")
                return {'result': 'failure', 'reason': 'No public key found'}
            publickeyfile = open(publickeyfile).read()
            conn.import_key_pair(KeyName=keypair, PublicKeyMaterial=publickeyfile)
        if cloudinit:
            if image is not None and common.needs_ignition(image):
                version = common.ignition_version(image)
                userdata = common.ignition(name=name, keys=keys, cmds=cmds, nets=nets, gateway=gateway, dns=dns,
                                           domain=domain, files=files, enableroot=enableroot,
                                           overrides=overrides, version=version, plan=plan, image=image,
                                           vmuser=vmuser)
            else:
                userdata = common.cloudinit(name=name, keys=keys, cmds=cmds, nets=nets, gateway=gateway, dns=dns,
                                            domain=domain, files=files, enableroot=enableroot,
                                            overrides=overrides, fqdn=True, storemetadata=storemetadata,
                                            vmuser=vmuser)[0]
        else:
            userdata = ''
        networkinterfaces = []
        blockdevicemappings = []
        privateips = []
        vpcs = conn.describe_vpcs()
        subnets = conn.describe_subnets()
        for index, net in enumerate(nets):
            networkinterface = {'DeleteOnTermination': True, 'Description': f"eth{index}", 'DeviceIndex': index,
                                'Groups': ['string'], 'SubnetId': 'string'}
            ip = None
            if isinstance(net, str):
                netname = net
                netpublic = True
            elif isinstance(net, dict) and 'name' in net:
                netname = net['name']
                ip = net.get('ip')
                alias = net.get('alias')
                netpublic = net.get('public', True)
            # if securitygroups:
            #    netpublic = False
            networkinterface['AssociatePublicIpAddress'] = netpublic if index == 0 else False
            if netname in [subnet['SubnetId'] for subnet in subnets['Subnets']]:
                vpcid = [subnet['VpcId'] for subnet in subnets['Subnets'] if subnet['SubnetId'] == netname][0]
            elif netname == 'default':
                if defaultsubnetid is not None:
                    netname = defaultsubnetid
                else:
                    vpcid = [vpc['VpcId'] for vpc in vpcs['Vpcs'] if vpc['IsDefault']][0]
                    subnetid = [subnet['SubnetId'] for subnet in subnets['Subnets']
                                if subnet['DefaultForAz'] and subnet['VpcId'] == vpcid][0]
                    netname = subnetid
                    defaultsubnetid = netname
                    pprint(f"Using subnet {defaultsubnetid} as default")
            else:
                vpcid = self.get_vpc_id(vpcs, netname) if not netname.startswith('vpc-') else netname
                if vpcid is None:
                    error(f"Couldn't find vpc {netname}")
                    sys.exit(1)
                subnetids = [subnet['SubnetId'] for subnet in subnets['Subnets'] if subnet['VpcId'] == vpcid]
                if subnetids:
                    netname = subnetids[0]
                else:
                    error(f"Couldn't find valid subnet for vpc {netname}")
                    sys.exit(1)
            if ips and len(ips) > index and ips[index] is not None:
                ip = ips[index]
                if index == 0:
                    networkinterface['PrivateIpAddress'] = ip
                    privateip = {'Primary': True, 'PrivateIpAddress': ip}
                else:
                    privateip = {'Primary': False, 'PrivateIpAddress': ip}
                privateips.append(privateip)
            networkinterface['SubnetId'] = netname
            if index == 0:
                SecurityGroupIds = []
                for sg in securitygroups:
                    sgid = self.get_security_group_id(sg, vpcid)
                    if sgid is not None:
                        SecurityGroupIds.append(sgid)
                if 'kubetype' in metadata and metadata['kubetype'] == "openshift":
                    kube = metadata['kube']
                    pprint(f"Adding vm to security group {kube}")
                    kubesgid = self.get_security_group_id(kube, vpcid)
                    if kubesgid is None:
                        sg = self.resource.create_security_group(GroupName=kube, Description=kube, VpcId=vpcid)
{'fraction_non_alphanumeric': 0.11819140774836977, 'fraction_numerical': 0.010644418872266973, 'mean_word_length': 2.0271407837445574, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4204682', 'n_tokens_mistral': 6256, 'n_tokens_neox': 5926, 'n_words': 1406}