<commit_before>#!/usr/bin/python
#
# Copyright (C) Citrix Systems Inc.
#
# This program is free software; you can redistribute it and/or modify 
# it under the terms of the GNU Lesser General Public License as published 
# by the Free Software Foundation; version 2.1 only.
#
# This program is distributed in the hope that it will be useful, 
# but WITHOUT ANY WARRANTY; without even the implied warranty of 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
# Miscellaneous utility functions
#

import os, re, sys, subprocess, shutil, tempfile, commands, signal
import time, datetime
import errno, socket
import xml.dom.minidom
import scsiutil
import statvfs
import stat
import xs_errors
import XenAPI,xmlrpclib
import base64
import syslog
import resource
import exceptions
import traceback
import glob
import copy

NO_LOGGING_STAMPFILE='/etc/xensource/no_sm_log'

IORETRY_MAX = 20 # retries
IORETRY_PERIOD = 1.0 # seconds

LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE))
_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2
LOG_EMERG   = syslog.LOG_EMERG
LOG_ALERT   = syslog.LOG_ALERT
LOG_CRIT    = syslog.LOG_CRIT
LOG_ERR     = syslog.LOG_ERR
LOG_WARNING = syslog.LOG_WARNING
LOG_NOTICE  = syslog.LOG_NOTICE
LOG_INFO    = syslog.LOG_INFO
LOG_DEBUG   = syslog.LOG_DEBUG

ISCSI_REFDIR = '/var/run/sr-ref'

CMD_DD = "/bin/dd"

FIST_PAUSE_PERIOD = 30 # seconds

class SMException(Exception):
    """Base class for all SM exceptions for easier catching & wrapping in 
    XenError"""
    pass

class CommandException(SMException):
    def __init__(self, code, cmd = "", reason='exec failed'):
        self.code = code
        self.cmd = cmd
        self.reason = reason
        Exception.__init__(self, code)

class SRBusyException(SMException):
    """The SR could not be locked"""
    pass

def logException(tag):
    info = sys.exc_info()
    if info[0] == exceptions.SystemExit:
        # this should not be happening when catching "Exception", but it is
        sys.exit(0)
    tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2]))
    str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb)
    SMlog(str)

def roundup(divisor, value):
    """Retruns the rounded up value so it is divisible by divisor."""

    if value == 0:
        value = 1
    if value % divisor != 0:
        return ((value / divisor) + 1) * divisor
    return value

def to_plain_string(obj):
    if obj is None:
        return None
    if type(obj) == str:
        return obj
    if type(obj) == unicode:
        return obj.encode("utf-8")
    return str(obj)

def shellquote(arg):
    return '"%s"' % arg.replace('"', '\\"')

def make_WWN(name):
    hex_prefix = name.find("0x")
    if (hex_prefix >=0):
        name = name[name.find("0x")+2:len(name)]
    # inject dashes for each nibble
    if (len(name) == 16): #sanity check
        name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \
               name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \
               name[12:14] + "-" + name[14:16]
    return name

def _logToSyslog(ident, facility, priority, message):
    syslog.openlog(ident, 0, facility)
    syslog.syslog(priority, "[%d] %s" % (os.getpid(), message))
    syslog.closelog()

def SMlog(message, ident="SM", priority=LOG_INFO):
    if LOGGING:
        for message_line in str(message).split('\n'):
            _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line)

def _getDateString():
    d = datetime.datetime.now()
    t = d.timetuple()
    return "%s-%s-%s:%s:%s:%s" % \
          (t[0],t[1],t[2],t[3],t[4],t[5])

def doexec(args, inputtext=None):
    """Execute a subprocess, then return its return code, stdout and stderr"""
    proc = subprocess.Popen(args,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,close_fds=True)
    (stdout,stderr) = proc.communicate(inputtext)
    rc = proc.returncode
    return (rc,stdout,stderr)
<commit_msg>CP-6167: Work around a pylint bug<commit_after>#!/usr/bin/python
#
# Copyright (C) Citrix Systems Inc.
#
# This program is free software; you can redistribute it and/or modify 
# it under the terms of the GNU Lesser General Public License as published 
# by the Free Software Foundation; version 2.1 only.
#
# This program is distributed in the hope that it will be useful, 
# but WITHOUT ANY WARRANTY; without even the implied warranty of 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
# Miscellaneous utility functions
#

import os, re, sys, subprocess, shutil, tempfile, commands, signal
import time, datetime
import errno, socket
import xml.dom.minidom
import scsiutil
import statvfs
import stat
import xs_errors
import XenAPI,xmlrpclib
import base64
import syslog
import resource
import exceptions
import traceback
import glob
import copy

NO_LOGGING_STAMPFILE='/etc/xensource/no_sm_log'

IORETRY_MAX = 20 # retries
IORETRY_PERIOD = 1.0 # seconds

LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE))
_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2
LOG_EMERG   = syslog.LOG_EMERG
LOG_ALERT   = syslog.LOG_ALERT
LOG_CRIT    = syslog.LOG_CRIT
LOG_ERR     = syslog.LOG_ERR
LOG_WARNING = syslog.LOG_WARNING
LOG_NOTICE  = syslog.LOG_NOTICE
LOG_INFO    = syslog.LOG_INFO
LOG_DEBUG   = syslog.LOG_DEBUG

ISCSI_REFDIR = '/var/run/sr-ref'

CMD_DD = "/bin/dd"

FIST_PAUSE_PERIOD = 30 # seconds

class SMException(Exception):
    """Base class for all SM exceptions for easier catching & wrapping in 
    XenError"""
    pass

class CommandException(SMException):
    def __init__(self, code, cmd = "", reason='exec failed'):
        self.code = code
        self.cmd = cmd
        self.reason = reason
        Exception.__init__(self, code)

class SRBusyException(SMException):
    """The SR could not be locked"""
    pass

def logException(tag):
    info = sys.exc_info()
    if info[0] == exceptions.SystemExit:
        # this should not be happening when catching "Exception", but it is
        sys.exit(0)
    tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2]))
    str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb)
    SMlog(str)

def roundup(divisor, value):
    """Retruns the rounded up value so it is divisible by divisor."""

    if value == 0:
        value = 1
    if value % divisor != 0:
        return ((value / divisor) + 1) * divisor
    return value

def to_plain_string(obj):
    if obj is None:
        return None
    if type(obj) == str:
        return obj
    if type(obj) == unicode:
        return obj.encode("utf-8")
    return str(obj)

def shellquote(arg):
    return '"%s"' % arg.replace('"', '\\"')

def make_WWN(name):
    hex_prefix = name.find("0x")
    if (hex_prefix >=0):
        name = name[name.find("0x")+2:len(name)]
    # inject dashes for each nibble
    if (len(name) == 16): #sanity check
        name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \
               name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \
               name[12:14] + "-" + name[14:16]
    return name

def _logToSyslog(ident, facility, priority, message):
    syslog.openlog(ident, 0, facility)
    syslog.syslog(priority, "[%d] %s" % (os.getpid(), message))
    syslog.closelog()

def SMlog(message, ident="SM", priority=LOG_INFO):
    if LOGGING:
        for message_line in str(message).split('\n'):
            _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line)

def _getDateString():
    d = datetime.datetime.now()
    t = d.timetuple()
    return "%s-%s-%s:%s:%s:%s" % \
          (t[0],t[1],t[2],t[3],t[4],t[5])

def doexec(args, inputtext=None):
    """Execute a subprocess, then return its return code, stdout and stderr"""
    proc = subprocess.Popen(args,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,close_fds=True)
    (stdout,stderr) = proc.communicate(inputtext)
    # Workaround for a pylint bug, can be removed after upgrade to
    # python 3.x or maybe a newer version of pylint in the future
    stdout = str(stdout)
    stderr = str(stderr)
    rc = proc.returncode
    return (rc,stdout,stderr)

{'fraction_non_alphanumeric': 0.12359420289855072, 'fraction_numerical': 0.01634782608695652, 'mean_word_length': 3.434961439588689, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6562684', 'n_tokens_mistral': 2998, 'n_tokens_neox': 2901, 'n_words': 856}