<commit_before>#!/usr/bin/env python

'''

api.py: Singularity Hub helper functions for python

Copyright (c) 2016-2017, <NAME>. All rights reserved. 

"Singularity" Copyright (c) 2016, The Regents of the University of California,
through Lawrence Berkeley National Laboratory (subject to receipt of any
required approvals from the U.S. Dept. of Energy).  All rights reserved.
 
This software is licensed under a customized 3-clause BSD license.  Please
consult LICENSE file distributed with the sources of this project regarding
your rights to use or distribute this software.
 
NOTICE.  This Software was developed under funding from the U.S. Department of
Energy and the U.S. Government consequently retains certain rights. As such,
the U.S. Government has been granted for itself and others acting on its
behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software
to reproduce, distribute copies to the public, prepare derivative works, and
perform publicly and display publicly, and to permit other to do so. 

'''

import sys
import os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
sys.path.append('..') # parent directory

from shell import (
    remove_image_uri
)

from sutils import (
    add_http,
    is_number,
    read_file
)

from base import ApiConnection

from helpers.json.main import ADD

from defaults import (
    SHUB_API_BASE
)

from message import bot
import json
import re

try:
    from urllib import unquote 
except:
    from urllib.parse import unquote


# Shub API Class  ----------------------------------------------------------------------------

class SingularityApiConnection(ApiConnection):

    def __init__(self,**kwargs):
        self.token = None
        self.api_base = SHUB_API_BASE
        if 'image' in kwargs:
            self.load_image(kwargs['image'])
        if 'token' in kwargs:
            self.token = kwargs['token']
        super(SingularityApiConnection, self).__init__(**kwargs)


    def load_image(self,image):
        self.image = image
        if not is_number(image):
            self.image = remove_image_uri(image,quiet=True)
          


    def get_manifest(self,image=None,registry=None):
        '''get_image will return a json object with image metadata, based on a unique id.
        :param image: the image name, either an id, or a repo name, tag, etc.
        :param registry: the registry (hub) to use, if not defined, default is used
        '''
        if image == None:
            image = self.image
        if registry == None:
            registry = self.api_base
        registry = add_http(registry) # make sure we have a complete url

        # Numeric images have slightly different endpoint from named
        if is_number(image) == True:
            base = "%s/containers/%s" %(registry,image)
        else:
            base = "%s/container/%s" %(registry,image)

        # ---------------------------------------------------------------
        # If we eventually have private images, need to authenticate here       
        # --------------------------------------------------------------- 

        response = self.get(base)
        try:
            response = json.loads(response)
        except:
            print("Error getting image manifest using url %s" %(base))
            sys.exit(1)
        return response


    def download_image(self,manifest,download_folder=None,extract=True):
        '''download_image will download a singularity image from singularity
        hub to a download_folder, named based on the image version (commit id)
        :param manifest: the manifest obtained with get_manifest
        :param download_folder: the folder to download to, if None, will be pwd
        :param extract: if True, will extract image to .img and return that.
        '''
        image_file = get_image_name(manifest)

        if not bot.is_quiet():
            print("Found image %s:%s" %(manifest['name'],
                                        manifest['branch']))

            print("Downloading image... %s" %(image_file))

        if download_folder is not None:
            image_file = "%s/%s" %(download_folder,image_file)
        url = manifest['image']

        # Download image file atomically, streaming
        image_file = self.download_atomically(url=url,
                                              file_name=image_file,
                                              suffix="downloading image")

        if extract == True:
            if not bot.is_quiet():
                print("Decompressing %s" %image_file)
            os.system('gzip -d -f %s' %(image_file))
            image_file = image_file.replace('.gz','')
        return image_file



# Various Helpers ---------------------------------------------------------------------------------
def get_image_name(manifest,extension='img.gz'):
    '''get_image_name will return the image name for a manifest
    :param manifest: the image manifest with 'image' as key with download link
    :param use_hash: use the image hash instead of name
    '''
    from defaults import ( SHUB_CONTAINERNAME, 
                           SHUB_NAMEBYCOMMIT, 
                           SHUB_NAMEBYHASH )<commit_msg>	modified:   libexec/python/shub/api.py<commit_after>#!/usr/bin/env python

'''

api.py: Singularity Hub helper functions for python

Copyright (c) 2016-2017, <NAME>. All rights reserved. 

"Singularity" Copyright (c) 2016, The Regents of the University of California,
through Lawrence Berkeley National Laboratory (subject to receipt of any
required approvals from the U.S. Dept. of Energy).  All rights reserved.
 
This software is licensed under a customized 3-clause BSD license.  Please
consult LICENSE file distributed with the sources of this project regarding
your rights to use or distribute this software.
 
NOTICE.  This Software was developed under funding from the U.S. Department of
Energy and the U.S. Government consequently retains certain rights. As such,
the U.S. Government has been granted for itself and others acting on its
behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software
to reproduce, distribute copies to the public, prepare derivative works, and
perform publicly and display publicly, and to permit other to do so. 

'''

import sys
import os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
sys.path.append('..') # parent directory

from shell import (
    remove_image_uri
)

from sutils import (
    add_http,
    is_number,
    read_file
)

from base import ApiConnection

from helpers.json.main import ADD

from defaults import (
    SHUB_API_BASE
)

from message import bot
import json
import re

try:
    from urllib import unquote 
except:
    from urllib.parse import unquote


# Shub API Class  ----------------------------------------------------------------------------

class SingularityApiConnection(ApiConnection):

    def __init__(self,**kwargs):
        self.token = <PASSWORD>
        self.api_base = SHUB_API_BASE
        if 'image' in kwargs:
            self.load_image(kwargs['image'])
        if 'token' in kwargs:
            self.token = kwargs['token']
        super(SingularityApiConnection, self).__init__(**kwargs)


    def load_image(self,image):
        self.image = image
        if not is_number(image):
            self.image = remove_image_uri(image,quiet=True)
          


    def get_manifest(self,image=None,registry=None):
        '''get_image will return a json object with image metadata, based on a unique id.
        :param image: the image name, either an id, or a repo name, tag, etc.
        :param registry: the registry (hub) to use, if not defined, default is used
        '''
        if image == None:
            image = self.image
        if registry == None:
            registry = self.api_base
        registry = add_http(registry) # make sure we have a complete url

        # Numeric images have slightly different endpoint from named
        if is_number(image) == True:
            base = "%s/containers/%s" %(registry,image)
        else:
            base = "%s/container/%s" %(registry,image)

        # ---------------------------------------------------------------
        # If we eventually have private images, need to authenticate here       
        # --------------------------------------------------------------- 

        response = self.get(base)
        try:
            response = json.loads(response)
        except:
            print("Error getting image manifest using url %s" %(base))
            sys.exit(1)
        return response


    def download_image(self,manifest,download_folder=None,extract=True):
        '''download_image will download a singularity image from singularity
        hub to a download_folder, named based on the image version (commit id)
        :param manifest: the manifest obtained with get_manifest
        :param download_folder: the folder to download to, if None, will be pwd
        :param extract: if True, will extract image to .img and return that.
        '''
        image_file = get_image_name(manifest)

        if not bot.is_quiet():
            print("Found image %s:%s" %(manifest['name'],
                                        manifest['branch']))

            print("Downloading image... %s" %(image_file))

        if download_folder is not None:
            image_file = "%s/%s" %(download_folder,image_file)
        url = manifest['image']

        # Download image file atomically, streaming
        image_file = self.download_atomically(url=url,
                                              file_name=image_file)

        if extract == True:
            if not bot.is_quiet():
                print("Decompressing %s" %image_file)
            os.system('gzip -d -f %s' %(image_file))
            image_file = image_file.replace('.gz','')
        return image_file



# Various Helpers ---------------------------------------------------------------------------------
def get_image_name(manifest,extension='img.gz'):
    '''get_image_name will return the image name for a manifest
    :param manifest: the image manifest with 'image' as key with download link
    :param use_hash: use the image hash instead of name
    '''
    from defaults import ( SHUB_CONTAINERNAME, 
                           SHUB_NAMEBYCOMMIT, 
                           SHUB_NAMEBYHASH )
{'fraction_non_alphanumeric': 0.13197239792984475, 'fraction_numerical': 0.0026835345984282154, 'mean_word_length': 2.080897549453794, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4196849', 'n_tokens_mistral': 2733, 'n_tokens_neox': 2616, 'n_words': 962}