<commit_before>'''

main.py: Singularity Hub helper functions for Singularity in Python

Copyright (c) 2016-2017, <NAME>. All rights reserved. 

"Singularity" Copyright (c) 2016, The Regents of the University of California,
through Lawrence Berkeley National Laboratory (subject to receipt of any
required approvals from the U.S. Dept. of Energy).  All rights reserved.
 
This software is licensed under a customized 3-clause BSD license.  Please
consult LICENSE file distributed with the sources of this project regarding
your rights to use or distribute this software.
 
NOTICE.  This Software was developed under funding from the U.S. Department of
Energy and the U.S. Government consequently retains certain rights. As such,
the U.S. Government has been granted for itself and others acting on its
behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software
to reproduce, distribute copies to the public, prepare derivative works, and
perform publicly and display publicly, and to permit other to do so. 

'''

import sys
import os
sys.path.append('..') # parent directory

from shell import parse_image_uri

from .api import (
    extract_metadata,
    get_image_name,
    SingularityApiConnection
)

from sutils import (
    add_http,
    get_cache,
    write_file
)

from defaults import SHUB_PREFIX

from logman import logger
import json
import re
import os
import tempfile


def SIZE(image,contentfile=None):
    '''size is intended to be run before an import, to return to the contentfile a list of sizes
    (one per layer) corresponding with the layers that will be downloaded for image
    '''
    logger.debug("Starting Singularity Hub SIZE, will get size from manifest")
    logger.info("Singularity Hub image: %s", image)
    client = SingularityApiConnection(image=image)
    manifest = client.get_manifest()
    size = json.loads(manifest['metrics'].replace("'",'"'))['size']
    if contentfile is not None:
        write_file(contentfile,str(size),mode="w")
    return size 


def PULL(image,download_folder=None,layerfile=None):
    '''PULL will retrieve a Singularity Hub image and download to the local file
    system, to the variable specified by SINGULARITY_PULLFOLDER.
    :param image: the singularity hub image name
    :param download folder: the folder to pull the image to.
    :param layerfile: if defined, write pulled image to file
    '''
    client = SingularityApiConnection(image=image)
    manifest = client.get_manifest()
    
    if download_folder == None:
        cache_base = get_cache(subfolder="shub")
    else:
        cache_base = download_folder

    # The image name is the md5 hash, download if it's not there
    image_name = get_image_name(manifest)
    image_file = "%s/%s" %(cache_base,image_name)
    if not os.path.exists(image_file):
        image_file = client.download_image(manifest=manifest,
                                           download_folder=cache_base)
    else:
        if logger.level > 0:
            print("Image already exists at %s, skipping download." %image_file)
    logger.info("Singularity Hub Image Download: %s", image_file)

    manifest = {'image_file': image_file,
                'manifest': manifest,
                'cache_base': cache_base,
                'image': image }

    if layerfile != None:
        logger.debug("Writing Singularity Hub image path to %s", layerfile)
        write_file(layerfile,image_file,mode="w")

    return manifest



def IMPORT(image,layerfile,labelfile=None):
    '''IMPORT takes one more step than ADD, returning the image written to a layerfile
    plus metadata written to the metadata base in rootfs.
    :param image: the singularity hub image name
    '''
    manifest = PULL(image,layerfile=layerfile)

    # Write metadata to base    
    manifest['metadata'] = extract_metadata(manifest=manifest['manifest'],
                                            labelfile=labelfile,
                                            prefix="%s_" %SHUB_PREFIX)<commit_msg>	modified:   libexec/python/shub/main.py<commit_after>'''

main.py: Singularity Hub helper functions for Singularity in Python

Copyright (c) 2016-2017, <NAME>. All rights reserved. 

"Singularity" Copyright (c) 2016, The Regents of the University of California,
through Lawrence Berkeley National Laboratory (subject to receipt of any
required approvals from the U.S. Dept. of Energy).  All rights reserved.
 
This software is licensed under a customized 3-clause BSD license.  Please
consult LICENSE file distributed with the sources of this project regarding
your rights to use or distribute this software.
 
NOTICE.  This Software was developed under funding from the U.S. Department of
Energy and the U.S. Government consequently retains certain rights. As such,
the U.S. Government has been granted for itself and others acting on its
behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software
to reproduce, distribute copies to the public, prepare derivative works, and
perform publicly and display publicly, and to permit other to do so. 

'''

import sys
import os
sys.path.append('..') # parent directory

from shell import parse_image_uri

from .api import (
    extract_metadata,
    get_image_name,
    SingularityApiConnection
)

from sutils import (
    add_http,
    get_cache,
    write_file
)

from defaults import SHUB_PREFIX

from logman import logger
import json
import re
import os
import tempfile


def SIZE(image,contentfile=None):
    '''size is intended to be run before an import, to return to the contentfile a list of sizes
    (one per layer) corresponding with the layers that will be downloaded for image
    '''
    logger.debug("Starting Singularity Hub SIZE, will get size from manifest")
    logger.info("Singularity Hub image: %s", image)
    client = SingularityApiConnection(image=image)
    manifest = client.get_manifest()
    size = json.loads(manifest['metrics'].replace("'",'"'))['size']
    if contentfile is not None:
        write_file(contentfile,str(size),mode="w")
    return size 


def PULL(image,download_folder=None,layerfile=None):
    '''PULL will retrieve a Singularity Hub image and download to the local file
    system, to the variable specified by SINGULARITY_PULLFOLDER.
    :param image: the singularity hub image name
    :param download folder: the folder to pull the image to.
    :param layerfile: if defined, write pulled image to file
    '''
    client = SingularityApiConnection(image=image)
    manifest = client.get_manifest()
    
    if download_folder == None:
        cache_base = get_cache(subfolder="shub")
    else:
        cache_base = download_folder

    # The image name is the md5 hash, download if it's not there
    image_name = get_image_name(manifest)
    image_file = "%s/%s" %(cache_base,image_name)
    if not os.path.exists(image_file):
        image_file = client.download_image(manifest=manifest,
                                           download_folder=cache_base)
    else:
        logger.info("Image already exists at %s, skipping download." %image_file)
    logger.info("Singularity Hub Image Download: %s", image_file)

    manifest = {'image_file': image_file,
                'manifest': manifest,
                'cache_base': cache_base,
                'image': image }

    if layerfile != None:
        logger.debug("Writing Singularity Hub image path to %s", layerfile)
        write_file(layerfile,image_file,mode="w")

    return manifest



def IMPORT(image,layerfile,labelfile=None):
    '''IMPORT takes one more step than ADD, returning the image written to a layerfile
    plus metadata written to the metadata base in rootfs.
    :param image: the singularity hub image name
    '''
    manifest = PULL(image,layerfile=layerfile)

    # Write metadata to base    
    manifest['metadata'] = extract_metadata(manifest=manifest['manifest'],
                                            labelfile=labelfile,
                                            prefix="%s_" %SHUB_PREFIX)
{'fraction_non_alphanumeric': 0.07666417352281227, 'fraction_numerical': 0.0036150585888805784, 'mean_word_length': 3.2743740010655302, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5496026', 'n_tokens_mistral': 2217, 'n_tokens_neox': 2108, 'n_words': 828}