<commit_before>import os
import logging
import time
from ..database.DatabaseInterface import DatabaseInterface
from ..database.BackgroundProcess import BackgroundProcess
from ..common.MaterialsCommonsGlobusInterface import MaterialsCommonsGlobusInterface


class GlobusMCTransfer:
    def __init__(self, mc_user_id):
        self.log = logging.getLogger(__name__ + "." + self.__class__.__name__)
        self.log.info("init - started")
        self.mc_user_id = mc_user_id
        self.mc_globus_interface = None
        self.destination_path = None
        self.log.info("init - done")

    def transfer_and_await(self, status_record_id):
        self.log.info("Starting transfer_and_await")
        self.mc_globus_interface = MaterialsCommonsGlobusInterface(self.mc_user_id)
        self.mc_globus_interface.setup_transfer_clients()
        status_record = DatabaseInterface().update_status(status_record_id, BackgroundProcess.RUNNING)
        source_endpoint = status_record['extras']['globus_endpoint']
        source_path = status_record['extras']['globus_path']
        destination_endpoint = os.environ.get('MC_CONFIDENTIAL_CLIENT_ENDPOINT')
        destination_path = status_record['extras']['globus_destination_path']
        self.destination_path = destination_path
        user_label = status_record['extras']['label']

        task_id = self.mc_globus_interface.transfer(
            source_endpoint=source_endpoint, source_path=source_path,
            destination_endpoint=destination_endpoint, destination_path=destination_path,
            label=user_label
        )

        self.log.info("Starting await loop")
        status = "STARTED"
        while not status == "SUCCEEDED" and not status == "FAILED":
            task = self.mc_globus_interface.user_transfer_client.get_task(task_id)
            status = task['status']
            self.log.info("Current task status = {}".format(status))
            time.sleep(5)
        # self.mc_globus_interface.clear_user_access_rule(destination_path)
        results = {'status': status}
        return results

    def cleanup_on_error(self):
        self.mc_globus_interface.clear_user_access_rule(self.destination_path)<commit_msg>Added log statement on Globus Transfer task to indicate number of files/bytes transfered<commit_after>import os
import logging
import time
from ..database.DatabaseInterface import DatabaseInterface
from ..database.BackgroundProcess import BackgroundProcess
from ..common.MaterialsCommonsGlobusInterface import MaterialsCommonsGlobusInterface


class GlobusMCTransfer:
    def __init__(self, mc_user_id):
        self.log = logging.getLogger(__name__ + "." + self.__class__.__name__)
        self.log.info("init - started")
        self.mc_user_id = mc_user_id
        self.mc_globus_interface = None
        self.destination_path = None
        self.log.info("init - done")

    def transfer_and_await(self, status_record_id):
        self.log.info("Starting transfer_and_await")
        self.mc_globus_interface = MaterialsCommonsGlobusInterface(self.mc_user_id)
        self.mc_globus_interface.setup_transfer_clients()
        status_record = DatabaseInterface().update_status(status_record_id, BackgroundProcess.RUNNING)
        source_endpoint = status_record['extras']['globus_endpoint']
        source_path = status_record['extras']['globus_path']
        destination_endpoint = os.environ.get('MC_CONFIDENTIAL_CLIENT_ENDPOINT')
        destination_path = status_record['extras']['globus_destination_path']
        self.destination_path = destination_path
        user_label = status_record['extras']['label']

        task_id = self.mc_globus_interface.transfer(
            source_endpoint=source_endpoint, source_path=source_path,
            destination_endpoint=destination_endpoint, destination_path=destination_path,
            label=user_label
        )

        self.log.info("Starting await loop")
        status = "STARTED"
        while not status == "SUCCEEDED" and not status == "FAILED":
            task = self.mc_globus_interface.user_transfer_client.get_task(task_id)
            status = task['status']
            self.log.info("Transferred: bytes - {}, files - {}/{}".format(
                task['bytes_transferred'], task['files_transferred'], task['files']
            ))
            self.log.info("Current task status = {}".format(status))
            time.sleep(5)
        # self.mc_globus_interface.clear_user_access_rule(destination_path)
        results = {'status': status}
        return results

    def cleanup_on_error(self):
        self.mc_globus_interface.clear_user_access_rule(self.destination_path)
{'fraction_non_alphanumeric': 0.0928279129872927, 'fraction_numerical': 0.0004307559767391773, 'mean_word_length': 3.548481880509305, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4664574', 'n_tokens_mistral': 1308, 'n_tokens_neox': 1293, 'n_words': 212}