<commit_before>#!/bin/bash

import os
import pprint
import json
import time
import logging
import atexit

from azure.common.credentials import UserPassCredentials
from libsubmit.execution_provider_base import ExecutionProvider
from libsubmit.azure.azureDeployer import Deployer

translate_table = {'PD': 'PENDING',
                   'R': 'RUNNING',
                   'CA': 'CANCELLED',
                   'CF': 'PENDING',  # (configuring),
                   'CG': 'RUNNING',  # (completing),
                   'CD': 'COMPLETED',
                   'F': 'FAILED',  # (failed),
                   'TO': 'TIMEOUT',  # (timeout),
                   'NF': 'FAILED',  # (node failure),
                   'RV': 'FAILED',  # (revoked) and
                   'SE': 'FAILED'}  # (special exit state

template_string = """
cd ~
sudo apt-get update -y
sudo apt-get install -y python3 python3-pip ipython
sudo pip3 install ipyparallel parsl
"""


class AzureProvider(ExecutionProvider):
    def __init__(self, config):
        """Initialize Azure provider. Uses Azure python sdk to provide execution resources"""
        self.config = self.read_configs(config)
        self.config_logger()
        credentials = UserPassCredentials(
            self.config['username'], self.config['pass'])
        subscription_id = self.config['subscriptionId']

        # self.resource_client = ResourceManagementClient(credentials, subscription_id)
        # self.storage_client = StorageManagementClient(credentials, subscription_id)

        self.resource_group_name = 'my_resource_group'
        self.deployer = Deployer(
            subscription_id,
            self.resource_group_name,
            self.read_configs(config))

    def config_logger(self):
        """Configure Logger"""
        logger = logging.getLogger("AzureProvider")
        logger.setLevel(logging.INFO)
        if not os.path.isfile(self.config['logFile']):
            with open(self.config['logFile'], 'w') as temp_log:
                temp_log.write("Creating new log file.\n")
        fh = logging.FileHandler(self.config['logFile'])
        fh.setLevel(logging.INFO)
        formatter = logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        logger.addHandler(fh)
        self.logger = logger

    def _read_conf(self, config_file):
        """read config file"""
        config = json.load(open(config_file, 'r'))
        return config

    def pretty_configs(self, configs):
        """prettyprint config"""
        printer = pprint.PrettyPrinter(indent=4)
        printer.pprint(configs)

    def read_configs(self, config_file):
        """Read config file"""
        config = self._read_conf(config_file)
        return config

    def ipyparallel_configuration(self):
        config = ''
        try:
            with open(os.path.expanduser(self.config['iPyParallelConfigFile'])) as f:
                config = f.read().strip()
        except Exception as e:
            self.logger.error(e)
            self.logger.info(
                "Couldn't find user ipyparallel config file. Trying default location.")
            with open(os.path.expanduser("~/.ipython/profile_parallel/security/ipcontroller-engine.json")) as f:
                config = f.read().strip()
        else:
            self.logger.error(
                "Cannot find iPyParallel config file. Cannot proceed.")
            return -1
        ipptemplate = """
cat <<EOF> ipengine.json
{}
EOF

mkdir -p '.ipengine_logs'
sleep 5
ipengine --file=ipengine.json &> .ipengine_logs/ipengine.log""".format(config)
        return ipptemplate

    def submit(self):
        """Uses AzureDeployer to spin up an instance and connect it to the ipyparallel controller"""
        self.deployer.deploy()

    def status(self):
        """Get status of azure VM. Not implemented yet."""
        raise NotImplemented

    def cancel(self):
        """Destroy an azure VM"""
        self.deployer.destroy()

    def scale_in(self):
        raise NotImplemented

    def scale_out(self):
        raise NotImplemented


if __name__ == '__main__':
    config = "azureconf.json"
    provider = AzureProvider(config)
    provider.submit()
<commit_msg>Making Azure provider optional<commit_after>import os
import pprint
import json
import time
import logging
import atexit
from libsubmit.execution_provider_base import ExecutionProvider

try :
    from azure.common.credentials import UserPassCredentials
    from libsubmit.azure.azureDeployer import Deployer

except ImportError :
    _azure_enabled = False
else:
    _azure_enabled = True

translate_table = {'PD': 'PENDING',
                   'R': 'RUNNING',
                   'CA': 'CANCELLED',
                   'CF': 'PENDING',  # (configuring),
                   'CG': 'RUNNING',  # (completing),
                   'CD': 'COMPLETED',
                   'F': 'FAILED',  # (failed),
                   'TO': 'TIMEOUT',  # (timeout),
                   'NF': 'FAILED',  # (node failure),
                   'RV': 'FAILED',  # (revoked) and
                   'SE': 'FAILED'}  # (special exit state

template_string = """
cd ~
sudo apt-get update -y
sudo apt-get install -y python3 python3-pip ipython
sudo pip3 install ipyparallel parsl
"""


class AzureProvider(ExecutionProvider):
    def __init__(self, config):
        """Initialize Azure provider. Uses Azure python sdk to provide execution resources"""
        self.config = self.read_configs(config)
        self.config_logger()

        if not _azure_enabled :
            raise OptionalModuleMissing(['azure', 'haikunator'], "Azure Provider requires the azure and haikunator modules.")

        credentials = UserPassCredentials(
            self.config['username'], self.config['pass'])
        subscription_id = self.config['subscriptionId']

        # self.resource_client = ResourceManagementClient(credentials, subscription_id)
        # self.storage_client = StorageManagementClient(credentials, subscription_id)

        self.resource_group_name = 'my_resource_group'
        self.deployer = Deployer(
            subscription_id,
            self.resource_group_name,
            self.read_configs(config))

    def config_logger(self):
        """Configure Logger"""
        logger = logging.getLogger("AzureProvider")
        logger.setLevel(logging.INFO)
        if not os.path.isfile(self.config['logFile']):
            with open(self.config['logFile'], 'w') as temp_log:
                temp_log.write("Creating new log file.\n")
        fh = logging.FileHandler(self.config['logFile'])
        fh.setLevel(logging.INFO)
        formatter = logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        logger.addHandler(fh)
        self.logger = logger

    def _read_conf(self, config_file):
        """read config file"""
        config = json.load(open(config_file, 'r'))
        return config

    def pretty_configs(self, configs):
        """prettyprint config"""
        printer = pprint.PrettyPrinter(indent=4)
        printer.pprint(configs)

    def read_configs(self, config_file):
        """Read config file"""
        config = self._read_conf(config_file)
        return config

    def ipyparallel_configuration(self):
        config = ''
        try:
            with open(os.path.expanduser(self.config['iPyParallelConfigFile'])) as f:
                config = f.read().strip()
        except Exception as e:
            self.logger.error(e)
            self.logger.info(
                "Couldn't find user ipyparallel config file. Trying default location.")
            with open(os.path.expanduser("~/.ipython/profile_parallel/security/ipcontroller-engine.json")) as f:
                config = f.read().strip()
        else:
            self.logger.error(
                "Cannot find iPyParallel config file. Cannot proceed.")
            return -1
        ipptemplate = """
cat <<EOF> ipengine.json
{}
EOF

mkdir -p '.ipengine_logs'
sleep 5
ipengine --file=ipengine.json &> .ipengine_logs/ipengine.log""".format(config)
        return ipptemplate

    def submit(self):
        """Uses AzureDeployer to spin up an instance and connect it to the ipyparallel controller"""
        self.deployer.deploy()

    def status(self):
        """Get status of azure VM. Not implemented yet."""
        raise NotImplemented

    def cancel(self):
        """Destroy an azure VM"""
        self.deployer.destroy()

    def scale_in(self):
        raise NotImplemented

    def scale_out(self):
        raise NotImplemented


if __name__ == '__main__':
    config = "azureconf.json"
    provider = AzureProvider(config)
    provider.submit()

{'fraction_non_alphanumeric': 0.11311906668191696, 'fraction_numerical': 0.0013725265927027337, 'mean_word_length': 2.390461419154711, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '224303', 'n_tokens_mistral': 2511, 'n_tokens_neox': 2436, 'n_words': 598}