<commit_before># glint utils

import logging
import os

from keystoneclient.auth.identity import v2, v3
from keystoneauth1 import session
from keystoneauth1 import exceptions
from novaclient import client as novaclient
import glanceclient

from cloudscheduler.lib.db_config import Config
config = Config('/etc/cloudscheduler/cloudscheduler.yaml', ['general', 'openstackPoller.py', 'web_frontend'], pool_size=2, max_overflow=10)


def get_nova_client(session, region=None):
    nova = novaclient.Client("2", session=session, region_name=region, timeout=10)
    return nova

def get_glance_client(session, region=None):
    glance = glanceclient.Client("2", session=session, region_name=region)
    return glance

def get_openstack_session(cloud):
    authsplit = cloud.authurl.split('/')
    try:
        version = int(float(authsplit[-1][1:])) if len(authsplit[-1]) > 0 else int(float(authsplit[-2][1:]))
    except ValueError:
        logging.debug("Bad OpenStack URL, could not determine version, skipping %s", cloud.authurl)
        return False
    if version == 2:
        session = _get_openstack_session_v1_v2(
            auth_url=cloud.authurl,
            username=cloud.username,
            password=<PASSWORD>,
            project=cloud.project)
    else:
        session = _get_openstack_session_v1_v2(
            auth_url=cloud.authurl,
            username=cloud.username,
            password=<PASSWORD>,
            project=cloud.project,
            user_domain=cloud.user_domain_name,
            project_domain_name=cloud.project_domain_name,
            project_domain_id=cloud.project_domain_id,)
    if session is False:
        logging.error("Failed to setup session, skipping %s", cloud.cloud_name)
        if version == 2:
            logging.error("Connection parameters: \n authurl: %s \n username: %s \n project: %s",
                          (cloud.authurl, cloud.username, cloud.project))
        else:
            logging.error(
                "Connection parameters: \n authurl: %s \n username: %s \n project: %s \n user_domain: %s \n project_domain: %s",
                (cloud.authurl, cloud.username, cloud.project, cloud.user_domain, cloud.project_domain_name))
    return session

def _get_openstack_session_v1_v2(auth_url, username, password, project, user_domain="Default", project_domain_name="Default",
                                 project_domain_id=None):
    authsplit = auth_url.split('/')
    try:
        version = int(float(authsplit[-1][1:])) if len(authsplit[-1]) > 0 else int(float(authsplit[-2][1:]))
    except ValueError:
        logging.debug("Bad openstack URL: %s, could not determine version, aborting session", auth_url)
        return False
    if version == 2:
        try:
            auth = v2.Password(
                auth_url=auth_url,
                username=username,
                password=password,
                tenant_name=project)
            sess = session.Session(auth=auth, verify=config.cacerts)
        except Exception as exc:
            logging.error("Problem importing keystone modules, and getting session for grp:cloud - %s::%s" % (auth_url, exc))
            logging.error("Connection parameters: \n authurl: %s \n username: %s \n project: %s", (auth_url, username, project))
            return False
        return sess
    elif version == 3:
        #connect using keystone v3
        try:
            auth = v3.Password(
                auth_url=auth_url,
                username=username,
                password=password,
                project_name=project,
                user_domain_name=user_domain,
                project_domain_name=project_domain_name,
                project_domain_id=project_domain_id,)
            sess = session.Session(auth=auth, verify=config.cacerts)
        except Exception as exc:
            logging.error("Problem importing keystone modules, and getting session for grp:cloud - %s: %s", exc)
            logging.error("Connection parameters: \n authurl: %s \n username: %s \n project: %s \n user_domain: %s \n project_domain: %s", (auth_url, username, project, user_domain, project_domain_name))
            return False
        return sess




# new glint api, maybe make this part of glint utils?

def create_placeholder_image(glance, image_name, disk_format, container_format):
    image = glance.images.create(
        name=image_name,
        disk_format=disk_format,
        container_format=container_format)
    return image.id


# Upload an image to repo, returns image id if successful
# if there is no image_id it is a direct upload and no placeholder exists
def upload_image(glance, image_id, image_name, scratch_dir, disk_format=None, container_format=None):
    if image_id is not None:
        #this is the 2nd part of a transfer not a direct upload
        file_path = scratch_dir + image_name
        glance.images.upload(image_id, open(file_path, 'rb'))
        return glance.images.get(image_id)

    else:
        #this is a straight upload not part of a transfer
        image = glance.images.create(
            name=image_name,
            disk_format=disk_format,
            container_format=container_format)
        glance.images.upload(image.id, open(scratch_dir, 'rb'))
        logging.info("Upload complete")
        return image 


# Download an image from the repo, returns True if successful or False if not
def download_image(glance, image_name, image_id, scratch_dir):
    #open file then write to it
    try:
        if not os.path.exists(scratch_dir):
            os.makedirs(scratch_dir)
        file_path = scratch_dir + image_name
        image_file = open(file_path, 'wb')
        for chunk in glance.images.data(image_id):
            image_file.write(bytes(chunk))
        img = glance.images.get(image_id)

        return (True, "Success", img.disk_format, img.container_format)
    except Exception as exc:
        return (False, exc, "", "")

def delete_image(glance, image_id):
    try:
        glance.images.delete(image_id)
    except Exception as exc:
        logging.error("Unknown error, unable to delete image")
        logging.error(exc)
        return (1, exc)
    return (0,)


def update_image_name(glance, image_id, image_name):
    glance.images.update(image_id, name=image_name)


def get_checksum(glance, image_id):
    image = glance.images.get(image_id)
    return image['checksum']


def delete_keypair(key_name, cloud):
    sess = get_openstack_session(cloud)
    nova = get_nova_client(sess, cloud.region)

    keys = nova.keypairs.list()
    for key in keys:
        if key.name == key_name:
            nova.keypairs.delete(key)
            return True

    return False

def get_keypair(keypair_key, cloud):
    sess = get_openstack_session(cloud)
    nova = get_nova_client(sess, cloud.region)

    split_key = keypair_key.split(";")<commit_msg>move check cache and generate_tx_id to glint_utils<commit_after># glint utils

import logging
import os
import string

from keystoneclient.auth.identity import v2, v3
from keystoneauth1 import session
from keystoneauth1 import exceptions
from novaclient import client as novaclient
import glanceclient


from .celery_app import pull_request

from cloudscheduler.lib.db_config import Config
config = Config('/etc/cloudscheduler/cloudscheduler.yaml', ['general', 'openstackPoller.py', 'web_frontend'], pool_size=2, max_overflow=10)
ALPHABET = string.ascii_letters + string.digits + string.punctuation

def get_nova_client(session, region=None):
    nova = novaclient.Client("2", session=session, region_name=region, timeout=10)
    return nova

def get_glance_client(session, region=None):
    glance = glanceclient.Client("2", session=session, region_name=region)
    return glance

def get_openstack_session(cloud):
    authsplit = cloud.authurl.split('/')
    try:
        version = int(float(authsplit[-1][1:])) if len(authsplit[-1]) > 0 else int(float(authsplit[-2][1:]))
    except ValueError:
        logging.debug("Bad OpenStack URL, could not determine version, skipping %s", cloud.authurl)
        return False
    if version == 2:
        session = _get_openstack_session_v1_v2(
            auth_url=cloud.authurl,
            username=cloud.username,
            password=<PASSWORD>,
            project=cloud.project)
    else:
        session = _get_openstack_session_v1_v2(
            auth_url=cloud.authurl,
            username=cloud.username,
            password=<PASSWORD>,
            project=cloud.project,
            user_domain=cloud.user_domain_name,
            project_domain_name=cloud.project_domain_name,
            project_domain_id=cloud.project_domain_id,)
    if session is False:
        logging.error("Failed to setup session, skipping %s", cloud.cloud_name)
        if version == 2:
            logging.error("Connection parameters: \n authurl: %s \n username: %s \n project: %s",
                          (cloud.authurl, cloud.username, cloud.project))
        else:
            logging.error(
                "Connection parameters: \n authurl: %s \n username: %s \n project: %s \n user_domain: %s \n project_domain: %s",
                (cloud.authurl, cloud.username, cloud.project, cloud.user_domain, cloud.project_domain_name))
    return session

def _get_openstack_session_v1_v2(auth_url, username, password, project, user_domain="Default", project_domain_name="Default",
                                 project_domain_id=None):
    authsplit = auth_url.split('/')
    try:
        version = int(float(authsplit[-1][1:])) if len(authsplit[-1]) > 0 else int(float(authsplit[-2][1:]))
    except ValueError:
        logging.debug("Bad openstack URL: %s, could not determine version, aborting session", auth_url)
        return False
    if version == 2:
        try:
            auth = v2.Password(
                auth_url=auth_url,
                username=username,
                password=password,
                tenant_name=project)
            sess = session.Session(auth=auth, verify=config.cacerts)
        except Exception as exc:
            logging.error("Problem importing keystone modules, and getting session for grp:cloud - %s::%s" % (auth_url, exc))
            logging.error("Connection parameters: \n authurl: %s \n username: %s \n project: %s", (auth_url, username, project))
            return False
        return sess
    elif version == 3:
        #connect using keystone v3
        try:
            auth = v3.Password(
                auth_url=auth_url,
                username=username,
                password=password,
                project_name=project,
                user_domain_name=user_domain,
                project_domain_name=project_domain_name,
                project_domain_id=project_domain_id,)
            sess = session.Session(auth=auth, verify=config.cacerts)
        except Exception as exc:
            logging.error("Problem importing keystone modules, and getting session for grp:cloud - %s: %s", exc)
            logging.error("Connection parameters: \n authurl: %s \n username: %s \n project: %s \n user_domain: %s \n project_domain: %s", (auth_url, username, project, user_domain, project_domain_name))
            return False
        return sess




# new glint api, maybe make this part of glint utils?

def create_placeholder_image(glance, image_name, disk_format, container_format):
    image = glance.images.create(
        name=image_name,
        disk_format=disk_format,
        container_format=container_format)
    return image.id


# Upload an image to repo, returns image id if successful
# if there is no image_id it is a direct upload and no placeholder exists
def upload_image(glance, image_id, image_name, scratch_dir, disk_format=None, container_format=None):
    if image_id is not None:
        #this is the 2nd part of a transfer not a direct upload
        file_path = scratch_dir + image_name
        glance.images.upload(image_id, open(file_path, 'rb'))
        return glance.images.get(image_id)

    else:
        #this is a straight upload not part of a transfer
        image = glance.images.create(
            name=image_name,
            disk_format=disk_format,
            container_format=container_format)
        glance.images.upload(image.id, open(scratch_dir, 'rb'))
        logging.info("Upload complete")
        return image 


# Download an image from the repo, returns True if successful or False if not
def download_image(glance, image_name, image_id, scratch_dir):
    #open file then write to it
    try:
        if not os.path.exists(scratch_dir):
            os.makedirs(scratch_dir)
        file_path = scratch_dir + image_name
        image_file = open(file_path, 'wb')
        for chunk in glance.images.data(image_id):
            image_file.write(bytes(chunk))
        img = glance.images.get(image_id)

        return (True, "Success", img.disk_format, img.container_format)
    except Exception as exc:
        return (False, exc, "", "")

def delete_image(glance, image_id):
    try:
        glance.images.delete(image_id)
    except Exception as exc:
        logging.error("Unknown error, unable to delete image")
        logging.error(exc)
        return (1, exc)
    return (0,)


def update_image_name(glance, image_id, image_name):
    glance.images.update(image_id, name=image_name)


def get_checksum(glance, image_id):
    image = glance.images.get(image_id)
    return image['checksum']

#
# Check image cache and queue up a pull request if target image is not present
#
def check_cache(config, image_name, image_checksum, group_name, user):
    IMAGE_CACHE = config.db_map.classes.csv2_image_cache
    db_session = config.db_session

    if image_checksum is not None:
        image = db_session.query(IMAGE_CACHE).filter(IMAGE_CACHE.image_name == image_name, IMAGE_CACHE.checksum == image_checksum)
    else:
        image = db_session.query(IMAGE_CACHE).filter(IMAGE_CACHE.image_name == image_name)

    if image.count() > 0:
        # we found something in the cache we can skip queueing a pull request
        return True
    else:
        logger.info("No image n cache, getting target image for pull request")
        # nothing in the cache lets queue up a pull request
        target_image = _get_image(config, image_name, image_checksum, group_name)
        if target_image is False:
            # unable to find target image
            logger.info("Unable to find target image")
            return False #maybe raise an error here

        PULL_REQ = config.db_map.classes.csv2_image_pull_requests
        # check if a pull request already exists for this image? or just let the workers sort it out?
        tx_id =  generate_tx_id()
        preq = {
            "tx_id": tx_id,
            "target_group_name": target_image.group_name,
            "target_cloud_name": target_image.cloud_name,
            "image_name": image_name,
            "image_id": target_image.id,
            "checksum": target_image.checksum,
            "status": "pending",
            "requester": user.username,
        }
        new_preq = PULL_REQ(**preq)
        db_session.merge(new_preq)
        db_session.commit()
        #pull_request.delay(tx_id = tx_id)
        pull_request.apply_async((tx_id,), queue='pull_requests')

        return True

# at a length of 16 with a 94 symbol alphabet we have a N/16^94 chance of a collision, pretty darn unlikely
def generate_tx_id(length=16):
    return ''.join(random.choice(ALPHABET) for i in range(length)) 

def delete_keypair(key_name, cloud):
    sess = get_openstack_session(cloud)
    nova = get_nova_client(sess, cloud.region)

    keys = nova.keypairs.list()
    for key in keys:
        if key.name == key_name:
            nova.keypairs.delete(key)
            return True

    return False

def get_keypair(keypair_key, cloud):
    sess = get_openstack_session(cloud)
    nova = get_nova_client(sess, cloud.region)

    split_key = keypair_key.split(";")
{'fraction_non_alphanumeric': 0.09328147544068753, 'fraction_numerical': 0.005708550279154382, 'mean_word_length': 2.7804126156035096, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4259846', 'n_tokens_mistral': 4762, 'n_tokens_neox': 4652, 'n_words': 1239}