<commit_before>#System import subprocess
import subprocess
import getopt
import sys

# PID imports
from b2handle.clientcredentials import PIDClientCredentials
from b2handle.handleclient import EUDATHandleClient

import uuid
import hashlib
import os, shutil

RED     = "\033[31m"
GREEN   = "\033[92m"
BLUE    = "\033[34m"
DEFAULT = "\033[0m"

#Upload dataset to gridFTP server
def gridftp_upload(dataset, server, protocol, destination):
    exit_code = subprocess.call(["grid-proxy-init"])
    if exit_code == 0:
        print GREEN, "DEBUG", DEFAULT, \
            "Uplopading", dataset, "to", protocol+"://"+server+destination
        exit_code = subprocess.call(["globus-url-copy", "-cd", "-r",
                    dataset, 
                    protocol+"://"+server+destination])
        print GREEN, "DEBUG"
        exit_code = subprocess.call(["globus-url-copy", "-list",
                    protocol+"://"+server+destination])
        print DEFAULT
    return exit_code

#Connect to handle server --> return client instance
def conn_handle(credentials='cred_21.T12995.json'):
    cred = PIDClientCredentials.load_from_JSON(credentials)
    print GREEN, "DEBUG"
    print('PID prefix ' + cred.get_prefix())
    print('Server ' + cred.get_server_URL())
    print DEFAULT    

    ec = EUDATHandleClient.instantiate_with_credentials(cred)

    return ec, cred

def register_dataset(ec, cred, dataset, protocol, server):
    #reverse lookup
    rev_args = dict([('URL', dataset)])
    if ec.search_handle(**rev_args)==[]:
        #Register dataset (folder) on gridftp server
        uid = uuid.uuid1()
        Handle  = ec.register_handle(cred.get_prefix() + '/' + str(uid), dataset)
        print GREEN, "DEBUG", DEFAULT, \
            "Creating handle", Handle, "for", dataset
        #Add information types
        args        = dict([('TYPE', 'Folder'), ('PROTOCOL', protocol), 
                    ('SITE', server)])
        exit_code   = ec.modify_handle_value(Handle, ttl=None, 
                    add_if_not_exist=True, **args)
    else:
        print RED, "WARNING", DEFAULT, dataset, \
                        "already has handles", ec.search_handle(**rev_args)
        Handle = ec.search_handle(**rev_args)[0]

    return Handle
 
def register_files(ec, cred, dataset, protocol, server):
    #Create PID for each file and subcollection in the dataset
    # List dataset
    args       = dict([('TYPE', 'Folder'), ('PROTOCOL', protocol),
                    ('SITE', server)])
    collection = [dataset]
    while len(collection) > 0:
        coll = collection[0]
        p = subprocess.Popen(["globus-url-copy -list "+ protocol+"://"+server+coll], 
            shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        for line in p.stdout.readlines()[1:]:
            if line.strip() == "":
                continue
            else:
                #Check if folder or file
                if line.strip().endswith('/'):
                    args["TYPE"]    = "Folder"
                    collection.append(coll+line.strip())
                    #print collection
                else:
                    args["TYPE"]    = "File"
                #reverse lookup, do not regenerate PIDs
                rev_args = dict([('URL', coll+line.strip())])
                if ec.search_handle(**rev_args)==[]:
                    uid = uuid.uuid1()
                    h   = ec.register_handle(cred.get_prefix() + '/' + str(uid), 
                        coll+line.strip())
                    exit_code       = ec.modify_handle_value(h, ttl=None, 
                            add_if_not_exist=True, **args)
                    print GREEN, "DEBUG", DEFAULT, "Created handle", h, \
                        "for", coll+line.strip()
                else:
                    print RED, "WARNING", DEFAULT, coll+line.strip(), \
                        "already has handles", ec.search_handle(**rev_args)
        collection.remove(coll)

def download_dataset(pid, destination):
    #Instantiate client for reading --> credentials necessary
    ec      = EUDATHandleClient.instantiate_for_read_access('https://hdl.handle.net')    
    record  = ec.retrieve_handle_record(pid)

    assert 'URL'        in record
    assert 'PROTOCOL'   in record
    assert 'SITE'       in record

    protocol    = record['PROTOCOL']
    site        = record['SITE']
    source      = record['URL']

    print GREEN, "DEBUG", DEFAULT, \<commit_msg>Adding Child-Parent relation on PID level.<commit_after>#System import subprocess
import subprocess
import getopt
import sys

# PID imports
from b2handle.clientcredentials import PIDClientCredentials
from b2handle.handleclient import EUDATHandleClient

import uuid
import hashlib
import os, shutil

RED     = "\033[31m"
GREEN   = "\033[92m"
BLUE    = "\033[34m"
DEFAULT = "\033[0m"

#Upload dataset to gridFTP server
def gridftp_upload(dataset, server, protocol, destination):
    exit_code = subprocess.call(["grid-proxy-init"])
    if exit_code == 0:
        print GREEN, "DEBUG", DEFAULT, \
            "Uplopading", dataset, "to", protocol+"://"+server+destination
        exit_code = subprocess.call(["globus-url-copy", "-cd", "-r",
                    dataset, 
                    protocol+"://"+server+destination])
        print GREEN, "DEBUG"
        exit_code = subprocess.call(["globus-url-copy", "-list",
                    protocol+"://"+server+destination])
        print DEFAULT
    return exit_code

#Connect to handle server --> return client instance
def conn_handle(credentials='cred_21.T12995.json'):
    cred = PIDClientCredentials.load_from_JSON(credentials)
    print GREEN, "DEBUG"
    print('PID prefix ' + cred.get_prefix())
    print('Server ' + cred.get_server_URL())
    print DEFAULT    

    ec = EUDATHandleClient.instantiate_with_credentials(cred)

    return ec, cred

def register_dataset(ec, cred, dataset, protocol, server):
    #reverse lookup
    rev_args = dict([('URL', dataset)])
    if ec.search_handle(**rev_args)==[]:
        #Register dataset (folder) on gridftp server
        uid = uuid.uuid1()
        Handle  = ec.register_handle(cred.get_prefix() + '/' + str(uid), dataset)
        print GREEN, "DEBUG", DEFAULT, \
            "Creating handle", Handle, "for", dataset
        #Add information types
        args        = dict([('TYPE', 'Folder'), ('PROTOCOL', protocol), 
                    ('SITE', server)])
        exit_code   = ec.modify_handle_value(Handle, ttl=None, 
                    add_if_not_exist=True, **args)
    else:
        print RED, "WARNING", DEFAULT, dataset, \
                        "already has handles", ec.search_handle(**rev_args)
        Handle = ec.search_handle(**rev_args)[0]

    return Handle

# Returns a list of children or empty list
def get_children(pid, ec):
    entry = ec.get_value_from_handle(pid, 'CHILDREN')
    if entry == None:
        return []
    else:
        return entry.replace("u'", "").replace("'", "").strip("]").strip("[").split(', ')    

def register_files(ec, cred, dataset, protocol, server):
    #Create PID for each file and subcollection in the dataset
    args       = dict([('TYPE', 'Folder'), ('PROTOCOL', protocol),
                    ('SITE', server)])
    parent_args = dict()
    collection = [dataset] # root collection
    while len(collection) > 0:
        children = []
        coll = collection[0]
        rev_args = dict([('URL', coll)])
        coll_pid = ec.search_handle(**rev_args)[0]
        p = subprocess.Popen(["globus-url-copy -list "+ protocol+"://"+server+coll], 
            shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        for line in p.stdout.readlines()[1:]: #get files and folders
            if line.strip() == "":
                continue
            else:
                #Check if folder or file
                if line.strip().endswith('/'):
                    args["TYPE"]    = "Folder"
                    collection.append(coll+line.strip())
                    #print collection
                else:
                    args["TYPE"]    = "File"
                args["PARENT"]  = coll_pid
                #reverse lookup for field URL, do not regenerate PIDs for same paths
                rev_args = dict([('URL', coll+line.strip())])
                if ec.search_handle(**rev_args)==[]:
                    uid = uuid.uuid1()
                    h   = ec.register_handle(cred.get_prefix() + '/' + str(uid), 
                        coll+line.strip())
                    children.append(h)
                    exit_code       = ec.modify_handle_value(h, ttl=None, 
                            add_if_not_exist=True, **args)
                    print GREEN, "DEBUG", DEFAULT, "Created handle", h, \
                        "for", coll+line.strip()
                else:
                    children.extend(ec.search_handle(**rev_args))
                    print RED, "WARNING", DEFAULT, coll+line.strip(), \
                        "already has handles", ec.search_handle(**rev_args)
        # Update collection with all PIDs to children
        parent_args['CHILDREN'] = ', '.join(children)
        print GREEN, "DEBUG", DEFAULT, "Update ", coll_pid
        #print GREEN, "DEBUG", DEFAULT, "CHILDREN ", children
        exit_code = ec.modify_handle_value(coll_pid, ttl=None, 
            add_if_not_exist=True, **parent_args)
        print exit_code
        collection.remove(coll)

def download_dataset(pid, destination):
    #Instantiate client for reading --> credentials necessary
    ec      = EUDATHandleClient.instantiate_for_read_access('https://hdl.handle.net')    
    record  = ec.retrieve_handle_record(pid)

    assert 'URL'        in record
    assert 'PROTOCOL'   in record
    assert 'SITE'       in record

    protocol    = record['PROTOCOL']
    site        = record['SITE']
    source      = record['URL']

    print GREEN, "DEBUG", DEFAULT, \
{'fraction_non_alphanumeric': 0.12666801455721796, 'fraction_numerical': 0.007177517185604529, 'mean_word_length': 2.1708333333333334, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6554625', 'n_tokens_mistral': 2926, 'n_tokens_neox': 2780, 'n_words': 682}