<commit_before># -*- coding: utf-8 -*-
# Copyright 2015 Spotify AB. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

from __future__ import unicode_literals

# import stdlib
import re
import os
import time
import uuid
import tempfile
from scp import SCPClient
import paramiko
import hashlib
from datetime import datetime

# import third party lib
from netaddr import IPAddress
from netaddr.core import AddrFormatError

from netmiko import ConnectHandler
from netmiko.ssh_exception import NetMikoTimeoutException

# import NAPALM Base
import napalm_base.helpers
from napalm_base import NetworkDriver
from napalm_base.utils import py23_compat
from napalm_base.exceptions import ConnectionException
from napalm_base.exceptions import MergeConfigException
from napalm_base.exceptions import CommandErrorException
from napalm_base.exceptions import ReplaceConfigException
import napalm_base.constants as c

UPTIME_KEY_MAP = {
    'kern_uptm_days': 'up_days',
    'kern_uptm_hrs': 'up_hours',
    'kern_uptm_mins': 'up_mins',
    'kern_uptm_secs': 'up_secs'
}


class NXOSSSHDriver(NetworkDriver):
    def __init__(self, hostname, username, password, timeout=60, optional_args=None):
        if optional_args is None:
            optional_args = {}
        self.hostname = hostname
        self.username = username
        self.password = password
        self.timeout = timeout
        self.up = False
        self.replace = True
        self.loaded = False
        self.fc = None
        self.changed = False
        self.replace_file = None
        self.merge_candidate = ''

        if optional_args is None:
            optional_args = {}

        # Netmiko possible arguments
        netmiko_argument_map = {
            'port': None,
            'verbose': False,
            'timeout': self.timeout,
            'global_delay_factor': 1,
            'use_keys': False,
            'key_file': None,
            'ssh_strict': False,
            'system_host_keys': False,
            'alt_host_keys': False,
            'alt_key_file': '',
            'ssh_config_file': None,
            'allow_agent': False
        }

        # Build dict of any optional Netmiko args
        self.netmiko_optional_args = {
            k: optional_args.get(k, v)
            for k, v in netmiko_argument_map.items()
        }

        self.port = optional_args.get('port', 22)
        self.sudo_pwd = optional_args.get('sudo_pwd', self.password)

    def open(self):
        try:
            self.device = ConnectHandler(device_type='cisco_nxos',
                                         host=self.hostname,
                                         username=self.username,
                                         password=<PASSWORD>,
                                         **self.netmiko_optional_args)
            self.device.enable()
        except NetMikoTimeoutException:
            raise ConnectionException('Cannot connect to {}'.format(self.hostname))

    def close(self):
        if self.changed:
            self._delete_file(self.backup_file)
        self.device.disconnect()
        self.device = None

    @staticmethod
    def _compute_timestamp(stupid_cisco_output):
        """
        Some fields such `uptime` are returned as: 23week(s) 3day(s)
        This method will determine the epoch of the event.
        e.g.: 23week(s) 3day(s) -> 1462248287
        """
        if not stupid_cisco_output or stupid_cisco_output == 'never':
            return -1.0

        things = {
            'second(s)': {
                'weight': 1
            },
            'minute(s)': {
                'weight': 60
            },
            'hour(s)': {
                'weight': 3600
            },
            'day(s)': {
                'weight': 24 * 3600
            },
            'week(s)': {
                'weight': 7 * 24 * 3600
            },
            'year(s)': {
                'weight': 365.25 * 24 * 3600
            }
        }

        things_keys = things.keys()
        for part in stupid_cisco_output.split():
            for key in things_keys:
                if key in part:
                    things[key]['count'] = napalm_base.helpers.convert(
                        int, part.replace(key, ''), 0)

        delta = sum([det.get('count', 0) * det.get('weight') for det in things.values()])
        return time.time() - delta

    @staticmethod
    def fix_checkpoint_string(string, filename):
        # used to generate checkpoint-like files
        pattern = '''!Command: Checkpoint cmd vdc 1'''

        if '!Command' in string:
            return re.sub('!Command.*', pattern.format(filename), string)
        else:
            return "{0}\n{1}".format(pattern.format(filename), string)

    def is_alive(self):
        return {
            'is_alive': self.device.remote_conn.transport.is_active()
            }

    def load_replace_candidate(self, filename=None, config=None):
        self._replace_candidate(filename, config)
        self.replace = True
        self.loaded = True

    def _get_flash_size(self):
        command = 'dir {}'.format('bootflash:')
        output = self.device.send_command(command)

        match = re.search(r'(\d+) bytes free', output)
        bytes_free = match.group(1)

        return int(bytes_free)

    def _enough_space(self, filename):
        flash_size = self._get_flash_size()
        file_size = os.path.getsize(filename)
        if file_size > flash_size:
            return False
        return True

    def _verify_remote_file_exists(self, dst, file_system='bootflash:'):
        command = 'dir {0}/{1}'.format(file_system, dst)
        output = self.device.send_command(command)
        if 'No such file' in output:
            raise ReplaceConfigException('Could not transfer file.')

    def _replace_candidate(self, filename, config):
        if not filename:
            file_content = self.fix_checkpoint_string(config, self.replace_file)
            filename = self._create_tmp_file(config)
        else:
            if not os.path.isfile(filename):
                raise ReplaceConfigException("File {} not found".format(filename))

        self.replace_file = filename
        with open(self.replace_file, 'r+') as f:
            file_content = f.read()
            file_content = self.fix_checkpoint_string(file_content, self.replace_file)
            f.write(file_content)

        if not self._enough_space(self.replace_file):
            msg = 'Could not transfer file. Not enough space on device.'
            raise ReplaceConfigException(msg)

        self._check_file_exists(self.replace_file)
        dest = os.path.basename(self.replace_file)
        full_remote_path = 'bootflash:{}'.format(dest)
        with paramiko.SSHClient() as ssh:
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(hostname=self.hostname, username=self.username, password=self.password)

            try:
                with SCPClient(ssh.get_transport()) as scp_client:
                    scp_client.put(self.replace_file, full_remote_path)
            except Exception:
                time.sleep(10)
                file_size = os.path.getsize(filename)
                temp_size = self._verify_remote_file_exists(dest)
                if int(temp_size) != int(file_size):
                    msg = ('Could not transfer file. There was an error '
                           'during transfer. Please make sure remote '
                           'permissions are set.')
                raise ReplaceConfigException(msg)
        self.config_replace = True
        if config and os.path.isfile(self.replace_file):
            os.remove(self.replace_file)

    def _file_already_exists(self, dst):
        dst_hash = self._get_remote_md5(dst)
        src_hash = self._get_local_md5(dst)
        if src_hash == dst_hash:
            return True
        return False

    def _check_file_exists(self, cfg_file):
        command = 'dir {}'.format(cfg_file)
        output = self.device.send_command(command)
        if 'No such file' in output:
            return False
        else:
            return self._file_already_exists(cfg_file)

    def _get_remote_md5(self, dst):
        command = 'show file {0} md5sum'.format(dst)
        return self.device.send_command(command).strip()

    def _get_local_md5(self, dst, blocksize=2**20):
        md5 = hashlib.md5()
        local_file = open(dst, 'rb')
        buf = local_file.read(blocksize)
        while buf:
            md5.update(buf)
            buf = local_file.read(blocksize)
        local_file.close()
        return md5.hexdigest()

    def load_merge_candidate(self, filename=None, config=None):
        self.replace = False
        self.loaded = True

        if not filename and not config:
            raise MergeConfigException('filename or config param must be provided.')

        self.merge_candidate += '\n'  # insert one extra line
        if filename is not None:
            with open(filename, "r") as f:
                self.merge_candidate += f.read()
        else:
            self.merge_candidate += config

    @staticmethod
    def _create_tmp_file(config):
        tmp_dir = tempfile.gettempdir()
        rand_fname = py23_compat.text_type(uuid.uuid4())
        filename = os.path.join(tmp_dir, rand_fname)
        with open(filename, 'wt') as fobj:
            fobj.write(config)
        return filename

    def _create_sot_file(self):
        """Create Source of Truth file to compare."""
        commands = ['terminal dont-ask', 'checkpoint file sot_file']
        self._send_config_commands(commands)

    def _get_diff(self):
        """Get a diff between running config and a proposed file."""
        diff = []
        self._create_sot_file()
        command = ('show diff rollback-patch file {0} file {1}'.format(
                   'sot_file', self.replace_file.split('/')[-1]))
        diff_out = self.device.send_command(command)
        try:
            diff_out = diff_out.split(
                '#Generating Rollback Patch')[1].replace(
                    'Rollback Patch is Empty', '').strip()
            for line in diff_out.splitlines():
                if line:
                    if line[0].strip() != '!' and line[0].strip() != '.':
                        diff.append(line.rstrip(' '))
        except (AttributeError, KeyError):
            raise ReplaceConfigException(
                'Could not calculate diff. It\'s possible the given file doesn\'t exist.')
        return '\n'.join(diff)

    def _get_merge_diff(self):
        diff = []
        running_config = self.get_config(retrieve='running')['running']
        running_lines = running_config.splitlines()
        for line in self.merge_candidate.splitlines():
            if line not in running_lines and line:
                if line[0].strip() != '!':
                    diff.append(line)
        return '\n'.join(diff)
        # the merge diff is not necessarily what needs to be loaded
        # for example under NTP, as the `ntp commit` command might be
        # alread configured, it is mandatory to be sent
        # otherwise it won't take the new configuration - see #59
        # https://github.com/napalm-automation/napalm-nxos/issues/59
        # therefore this method will return the real diff
        # but the merge_candidate will remain unchanged
        # previously: self.merge_candidate = '\n'.join(diff)

    def compare_config(self):
        if self.loaded:
            if not self.replace:
                return self._get_merge_diff()
                # return self.merge_candidate
            diff = self._get_diff()
            return diff
        return ''

    def _save(self, filename='startup-config'):
        command = 'copy run %s' % filename
        output = self.device.send_command(command)
        if 'complete' in output.lower():
            return True
        return False

    def _commit_merge(self):
        commands = [command for command in self.merge_candidate.splitlines() if command]
        output = self.device.send_config_set(commands)
        if 'Invalid command' in output:
            raise MergeConfigException('Error while applying config!')
        if not self._save():
            raise CommandErrorException('Unable to commit config!')

    def _save_config(self, filename):
        """Save the current running config to the given file."""
        command = 'checkpoint file {}'.format(filename)
        self.device.send_command(command)

    def _disable_confirmation(self):
        self._send_config_commands(['terminal dont-ask'])

    def _load_config(self):
        command = 'rollback running file {0}'.format(self.replace_file.split('/')[-1])
        self._disable_confirmation()
        output = self.device.send_command(command)
        if 'Rollback failed' in output or output == []:
            return False
        return True

    def commit_config(self):
        if self.loaded:
            self.backup_file = 'config_' + str(datetime.now()).replace(' ', '_')
            self._save_config(self.backup_file)
            if self.replace:
                if self._load_config() is False:
                    raise ReplaceConfigException
            else:
                try:
                    self._commit_merge()
                    self.merge_candidate = ''  # clear the merge buffer<commit_msg>Making consistent rollback change to SSH driver<commit_after># -*- coding: utf-8 -*-
# Copyright 2015 Spotify AB. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

from __future__ import unicode_literals

# import stdlib
import re
import os
import time
import uuid
import tempfile
from scp import SCPClient
import paramiko
import hashlib
from datetime import datetime

# import third party lib
from netaddr import IPAddress
from netaddr.core import AddrFormatError

from netmiko import ConnectHandler
from netmiko.ssh_exception import NetMikoTimeoutException

# import NAPALM Base
import napalm_base.helpers
from napalm_base import NetworkDriver
from napalm_base.utils import py23_compat
from napalm_base.exceptions import ConnectionException
from napalm_base.exceptions import MergeConfigException
from napalm_base.exceptions import CommandErrorException
from napalm_base.exceptions import ReplaceConfigException
import napalm_base.constants as c

UPTIME_KEY_MAP = {
    'kern_uptm_days': 'up_days',
    'kern_uptm_hrs': 'up_hours',
    'kern_uptm_mins': 'up_mins',
    'kern_uptm_secs': 'up_secs'
}


class NXOSSSHDriver(NetworkDriver):
    def __init__(self, hostname, username, password, timeout=60, optional_args=None):
        if optional_args is None:
            optional_args = {}
        self.hostname = hostname
        self.username = username
        self.password = password
        self.timeout = timeout
        self.up = False
        self.replace = True
        self.loaded = False
        self.fc = None
        self.changed = False
        self.replace_file = None
        self.merge_candidate = ''

        if optional_args is None:
            optional_args = {}

        # Netmiko possible arguments
        netmiko_argument_map = {
            'port': None,
            'verbose': False,
            'timeout': self.timeout,
            'global_delay_factor': 1,
            'use_keys': False,
            'key_file': None,
            'ssh_strict': False,
            'system_host_keys': False,
            'alt_host_keys': False,
            'alt_key_file': '',
            'ssh_config_file': None,
            'allow_agent': False
        }

        # Build dict of any optional Netmiko args
        self.netmiko_optional_args = {
            k: optional_args.get(k, v)
            for k, v in netmiko_argument_map.items()
        }

        self.port = optional_args.get('port', 22)
        self.sudo_pwd = optional_args.get('sudo_pwd', self.password)

    def open(self):
        try:
            self.device = ConnectHandler(device_type='cisco_nxos',
                                         host=self.hostname,
                                         username=self.username,
                                         password=self.password,
                                         **self.netmiko_optional_args)
            self.device.enable()
        except NetMikoTimeoutException:
            raise ConnectionException('Cannot connect to {}'.format(self.hostname))

    def close(self):
        if self.changed:
            self._delete_file(self.backup_file)
        self.device.disconnect()
        self.device = None

    @staticmethod
    def _compute_timestamp(stupid_cisco_output):
        """
        Some fields such `uptime` are returned as: 23week(s) 3day(s)
        This method will determine the epoch of the event.
        e.g.: 23week(s) 3day(s) -> 1462248287
        """
        if not stupid_cisco_output or stupid_cisco_output == 'never':
            return -1.0

        things = {
            'second(s)': {
                'weight': 1
            },
            'minute(s)': {
                'weight': 60
            },
            'hour(s)': {
                'weight': 3600
            },
            'day(s)': {
                'weight': 24 * 3600
            },
            'week(s)': {
                'weight': 7 * 24 * 3600
            },
            'year(s)': {
                'weight': 365.25 * 24 * 3600
            }
        }

        things_keys = things.keys()
        for part in stupid_cisco_output.split():
            for key in things_keys:
                if key in part:
                    things[key]['count'] = napalm_base.helpers.convert(
                        int, part.replace(key, ''), 0)

        delta = sum([det.get('count', 0) * det.get('weight') for det in things.values()])
        return time.time() - delta

    @staticmethod
    def fix_checkpoint_string(string, filename):
        # used to generate checkpoint-like files
        pattern = '''!Command: Checkpoint cmd vdc 1'''

        if '!Command' in string:
            return re.sub('!Command.*', pattern.format(filename), string)
        else:
            return "{0}\n{1}".format(pattern.format(filename), string)

    def is_alive(self):
        return {
            'is_alive': self.device.remote_conn.transport.is_active()
            }

    def load_replace_candidate(self, filename=None, config=None):
        self._replace_candidate(filename, config)
        self.replace = True
        self.loaded = True

    def _get_flash_size(self):
        command = 'dir {}'.format('bootflash:')
        output = self.device.send_command(command)

        match = re.search(r'(\d+) bytes free', output)
        bytes_free = match.group(1)

        return int(bytes_free)

    def _enough_space(self, filename):
        flash_size = self._get_flash_size()
        file_size = os.path.getsize(filename)
        if file_size > flash_size:
            return False
        return True

    def _verify_remote_file_exists(self, dst, file_system='bootflash:'):
        command = 'dir {0}/{1}'.format(file_system, dst)
        output = self.device.send_command(command)
        if 'No such file' in output:
            raise ReplaceConfigException('Could not transfer file.')

    def _replace_candidate(self, filename, config):
        if not filename:
            file_content = self.fix_checkpoint_string(config, self.replace_file)
            filename = self._create_tmp_file(config)
        else:
            if not os.path.isfile(filename):
                raise ReplaceConfigException("File {} not found".format(filename))

        self.replace_file = filename
        with open(self.replace_file, 'r+') as f:
            file_content = f.read()
            file_content = self.fix_checkpoint_string(file_content, self.replace_file)
            f.write(file_content)

        if not self._enough_space(self.replace_file):
            msg = 'Could not transfer file. Not enough space on device.'
            raise ReplaceConfigException(msg)

        self._check_file_exists(self.replace_file)
        dest = os.path.basename(self.replace_file)
        full_remote_path = 'bootflash:{}'.format(dest)
        with paramiko.SSHClient() as ssh:
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(hostname=self.hostname, username=self.username, password=self.password)

            try:
                with SCPClient(ssh.get_transport()) as scp_client:
                    scp_client.put(self.replace_file, full_remote_path)
            except Exception:
                time.sleep(10)
                file_size = os.path.getsize(filename)
                temp_size = self._verify_remote_file_exists(dest)
                if int(temp_size) != int(file_size):
                    msg = ('Could not transfer file. There was an error '
                           'during transfer. Please make sure remote '
                           'permissions are set.')
                raise ReplaceConfigException(msg)
        self.config_replace = True
        if config and os.path.isfile(self.replace_file):
            os.remove(self.replace_file)

    def _file_already_exists(self, dst):
        dst_hash = self._get_remote_md5(dst)
        src_hash = self._get_local_md5(dst)
        if src_hash == dst_hash:
            return True
        return False

    def _check_file_exists(self, cfg_file):
        command = 'dir {}'.format(cfg_file)
        output = self.device.send_command(command)
        if 'No such file' in output:
            return False
        else:
            return self._file_already_exists(cfg_file)

    def _get_remote_md5(self, dst):
        command = 'show file {0} md5sum'.format(dst)
        return self.device.send_command(command).strip()

    def _get_local_md5(self, dst, blocksize=2**20):
        md5 = hashlib.md5()
        local_file = open(dst, 'rb')
        buf = local_file.read(blocksize)
        while buf:
            md5.update(buf)
            buf = local_file.read(blocksize)
        local_file.close()
        return md5.hexdigest()

    def load_merge_candidate(self, filename=None, config=None):
        self.replace = False
        self.loaded = True

        if not filename and not config:
            raise MergeConfigException('filename or config param must be provided.')

        self.merge_candidate += '\n'  # insert one extra line
        if filename is not None:
            with open(filename, "r") as f:
                self.merge_candidate += f.read()
        else:
            self.merge_candidate += config

    @staticmethod
    def _create_tmp_file(config):
        tmp_dir = tempfile.gettempdir()
        rand_fname = py23_compat.text_type(uuid.uuid4())
        filename = os.path.join(tmp_dir, rand_fname)
        with open(filename, 'wt') as fobj:
            fobj.write(config)
        return filename

    def _create_sot_file(self):
        """Create Source of Truth file to compare."""
        commands = ['terminal dont-ask', 'checkpoint file sot_file']
        self._send_config_commands(commands)

    def _get_diff(self):
        """Get a diff between running config and a proposed file."""
        diff = []
        self._create_sot_file()
        command = ('show diff rollback-patch file {0} file {1}'.format(
                   'sot_file', self.replace_file.split('/')[-1]))
        diff_out = self.device.send_command(command)
        try:
            diff_out = diff_out.split(
                '#Generating Rollback Patch')[1].replace(
                    'Rollback Patch is Empty', '').strip()
            for line in diff_out.splitlines():
                if line:
                    if line[0].strip() != '!' and line[0].strip() != '.':
                        diff.append(line.rstrip(' '))
        except (AttributeError, KeyError):
            raise ReplaceConfigException(
                'Could not calculate diff. It\'s possible the given file doesn\'t exist.')
        return '\n'.join(diff)

    def _get_merge_diff(self):
        diff = []
        running_config = self.get_config(retrieve='running')['running']
        running_lines = running_config.splitlines()
        for line in self.merge_candidate.splitlines():
            if line not in running_lines and line:
                if line[0].strip() != '!':
                    diff.append(line)
        return '\n'.join(diff)
        # the merge diff is not necessarily what needs to be loaded
        # for example under NTP, as the `ntp commit` command might be
        # alread configured, it is mandatory to be sent
        # otherwise it won't take the new configuration - see #59
        # https://github.com/napalm-automation/napalm-nxos/issues/59
        # therefore this method will return the real diff
        # but the merge_candidate will remain unchanged
        # previously: self.merge_candidate = '\n'.join(diff)

    def compare_config(self):
        if self.loaded:
            if not self.replace:
                return self._get_merge_diff()
                # return self.merge_candidate
            diff = self._get_diff()
            return diff
        return ''

    def _save(self, filename='startup-config'):
        command = 'copy run %s' % filename
        output = self.device.send_command(command)
        if 'complete' in output.lower():
            return True
        return False

    def _commit_merge(self):
        commands = [command for command in self.merge_candidate.splitlines() if command]
        output = self.device.send_config_set(commands)
        if 'Invalid command' in output:
            raise MergeConfigException('Error while applying config!')
        if not self._save():
            raise CommandErrorException('Unable to commit config!')

    def _save_config(self, filename):
        """Save the current running config to the given file."""
        command = 'checkpoint file {}'.format(filename)
        self.device.send_command(command)

    def _disable_confirmation(self):
        self._send_config_commands(['terminal dont-ask'])

    def _load_config(self):
        command = 'rollback running file {0}'.format(self.replace_file.split('/')[-1])
        self._disable_confirmation()
        try:
            rollback_result = self.device.send_command(command)
        except Exception:
            return False
        if 'Rollback failed.' in rollback_result['msg'] or 'ERROR' in rollback_result:
            raise ReplaceConfigException(rollback_result['msg'])
        elif rollback_result == []:
            return False
        return True

    def commit_config(self):
        if self.loaded:
            self.backup_file = 'config_' + str(datetime.now()).replace(' ', '_')
            self._save_config(self.backup_file)
            if self.replace:
                if self._load_config() is False:
                    raise ReplaceConfigException
            else:
                try:
                    self._commit_merge()
                    self.merge_candidate = ''  # clear the merge buffer
{'fraction_non_alphanumeric': 0.0959153122326775, 'fraction_numerical': 0.007413743940690049, 'mean_word_length': 2.0872579225352115, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4198575', 'n_tokens_mistral': 7970, 'n_tokens_neox': 7630, 'n_words': 2012}