<commit_before>#!/usr/bin/env python
#
# helper_tools.py - Module to abstract away operations that require third-party
#                   helper software not part of a standard Python distro.
#
# April 2014, <NAME>
# Copyright (c) 2013-2014 the COT project developers.
# See the COPYRIGHT.txt file at the top-level directory of this distribution
# and at https://github.com/glennmatthews/cot/blob/master/COPYRIGHT.txt.
#
# This file is part of the Common OVF Tool (COT) project.
# It is subject to the license terms in the LICENSE.txt file found in the
# top-level directory of this distribution and at
# https://github.com/glennmatthews/cot/blob/master/LICENSE.txt. No part
# of COT, including this file, may be copied, modified, propagated, or
# distributed except according to the terms contained in the LICENSE.txt file.

import logging
import os
import re
import subprocess

from .data_validation import ValueUnsupportedError

logger = logging.getLogger('cot')

class HelperNotFoundError(OSError):
    """Error thrown when a helper program cannot be located."""

class HelperError(EnvironmentError):
    """Error thrown when a helper program exits with non-zero return code."""

def check_output(args):
    """Wrapper for subprocess.check_output.
    1) Raises a HelperNotFoundError if the command doesn't exist, instead of
       an OSError.
    2) Raises a HelperError if the command doesn't return 0 when run,
       instead of subprocess.CalledProcessError.
    3) Automatically redirects stderr to stdout, captures both, and generates
       a debug message with the stdout contents.
    """
    cmd = args[0]
    logger.debug("Calling {0}".format(" ".join(args)))
    try:
        stdout = (subprocess.check_output(args, stderr=subprocess.STDOUT)
                  .decode())
    except OSError as e:
        raise HelperNotFoundError(e.errno,
                                  "Unable to locate helper program '{0}'. "
                                  "Please check your $PATH.".format(cmd))
    except subprocess.CalledProcessError as e:
        raise HelperError(e.returncode,
                          "Helper program '{0}' exited with error {1}:\n"
                          "> {2}\n{3}".format(cmd, e.returncode,
                                              " ".join(args),
                                              e.output.decode()))
    logger.debug("{0} output:\n{1}".format(cmd, stdout))
    return stdout


def get_checksum(file_path, checksum_type):
    """Get the checksum of the given file.
    Supported checksum_type values are 'md5' and 'sha1'.
    """

    if checksum_type == 'md5':
        try:
            md5sum = check_output(['md5sum', file_path])
            # md5sum outputs something like:
            # 835a7493384047fde776d6e0f6ccb492  foo.ovf
            # We only want the first part of this string
            return md5sum.split(" ")[0]
        except HelperNotFoundError:
            md5sum = check_output(['md5', file_path])
            # md5 outputs something like:
            # MD5 (foo.ovf) = 835a7493384047fde776d6e0f6ccb492
            # We want the last part of this string, minus any trailing \n
            return md5sum.strip().split(" ")[-1]
    elif checksum_type == 'sha1':
        # Depending on the platform we can use either 'sha1sum' or 'shasum'
        try:
            sha1sum = check_output(['sha1sum', file_path])
        except HelperNotFoundError:
            sha1sum = check_output(['shasum', '-a', '1', file_path])
        # sha1sum and shasum both output something like:
        # 1309dfbf9a556def784fa3cc011a2b1f343024ba foo.ovf
        # We only want the first part of this string
        return sha1sum.split(" ")[0]
    else:
        raise ValueUnsupportedError("checksum type",
                                    checksum_type,
                                    "'md5' or 'sha1'")


def get_disk_format(file_path):
    """Returns a tuple (format, subformat) representing the given disk file's
    file format.

    format may be 'vmdk', 'raw', or 'qcow2'
    subformat may be None, or one of many strings for 'vmdk' files.
    """

    logger.info("Invoking qemu-img to determine disk format of {0}"
                .format(file_path))
    qemu_stdout = check_output(['qemu-img', 'info', file_path])<commit_msg>Use hashlib for get_checksum() instead of relying on external tools<commit_after>#!/usr/bin/env python
#
# helper_tools.py - Module to abstract away operations that require third-party
#                   helper software not part of a standard Python distro.
#
# April 2014, <NAME>
# Copyright (c) 2013-2014 the COT project developers.
# See the COPYRIGHT.txt file at the top-level directory of this distribution
# and at https://github.com/glennmatthews/cot/blob/master/COPYRIGHT.txt.
#
# This file is part of the Common OVF Tool (COT) project.
# It is subject to the license terms in the LICENSE.txt file found in the
# top-level directory of this distribution and at
# https://github.com/glennmatthews/cot/blob/master/LICENSE.txt. No part
# of COT, including this file, may be copied, modified, propagated, or
# distributed except according to the terms contained in the LICENSE.txt file.

import hashlib
import logging
import os
import re
import subprocess

from .data_validation import ValueUnsupportedError

logger = logging.getLogger('cot')

class HelperNotFoundError(OSError):
    """Error thrown when a helper program cannot be located."""

class HelperError(EnvironmentError):
    """Error thrown when a helper program exits with non-zero return code."""

def check_output(args):
    """Wrapper for subprocess.check_output.
    1) Raises a HelperNotFoundError if the command doesn't exist, instead of
       an OSError.
    2) Raises a HelperError if the command doesn't return 0 when run,
       instead of subprocess.CalledProcessError.
    3) Automatically redirects stderr to stdout, captures both, and generates
       a debug message with the stdout contents.
    """
    cmd = args[0]
    logger.debug("Calling {0}".format(" ".join(args)))
    try:
        stdout = (subprocess.check_output(args, stderr=subprocess.STDOUT)
                  .decode())
    except OSError as e:
        raise HelperNotFoundError(e.errno,
                                  "Unable to locate helper program '{0}'. "
                                  "Please check your $PATH.".format(cmd))
    except subprocess.CalledProcessError as e:
        raise HelperError(e.returncode,
                          "Helper program '{0}' exited with error {1}:\n"
                          "> {2}\n{3}".format(cmd, e.returncode,
                                              " ".join(args),
                                              e.output.decode()))
    logger.debug("{0} output:\n{1}".format(cmd, stdout))
    return stdout


def get_checksum(file_path, checksum_type):
    """Get the checksum of the given file.
    Supported checksum_type values are 'md5' and 'sha1'.
    """

    if checksum_type == 'md5':
        h = hashlib.md5()
    elif checksum_type == 'sha1':
        h = hashlib.sha1()
    else:
        raise ValueUnsupportedError("checksum type",
                                    checksum_type,
                                    "'md5' or 'sha1'")
    BLOCKSIZE = 65536
    with open(file_path, 'rb') as file_obj:
        while True:
            buf = file_obj.read(BLOCKSIZE)
            if len(buf) == 0:
                break
            h.update(buf)

    return h.hexdigest()


def get_disk_format(file_path):
    """Returns a tuple (format, subformat) representing the given disk file's
    file format.

    format may be 'vmdk', 'raw', or 'qcow2'
    subformat may be None, or one of many strings for 'vmdk' files.
    """

    logger.info("Invoking qemu-img to determine disk format of {0}"
                .format(file_path))
    qemu_stdout = check_output(['qemu-img', 'info', file_path])
{'fraction_non_alphanumeric': 0.09559939301972686, 'fraction_numerical': 0.020232675771370764, 'mean_word_length': 2.4046491605682307, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5151940', 'n_tokens_mistral': 2347, 'n_tokens_neox': 2217, 'n_words': 754}