<commit_before>#!/usr/bin/env python
#
# Azure Linux extension
#
# Linux Azure Diagnostic Extension (Current version is specified in manifest.xml)
# Copyright (c) Microsoft Corporation
# All rights reserved.
# MIT License
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
#  documentation files (the ""Software""), to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
#  permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
#  Software.
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
#  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
#  OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from misc_helpers import get_storage_endpoint_with_account, escape_nonalphanumerics
import os.path
import traceback
import xml.etree.ElementTree as ET
import Utils.ApplicationInsightsUtil as AIUtil
import Utils.LadDiagnosticUtil as LadUtil
import Utils.XmlUtil as XmlUtil
from Utils.lad_logging_config import LadLoggingConfig, copy_source_mdsdevent_elems, LadLoggingConfigException
import Providers.Builtin as BuiltIn


_mdsd_xml_template = """
<MonitoringManagement eventVersion="2" namespace="" timestamp="2017-03-27T19:45:00.000" version="1.0">
  <Accounts>
    <Account account="" isDefault="true" key="" moniker="moniker" tableEndpoint="" />
    <SharedAccessSignature account="" isDefault="true" key="" moniker="moniker" tableEndpoint="" />
  </Accounts>

  <Management defaultRetentionInDays="90" eventVolume="">
    <Identity>
      <IdentityComponent name="DeploymentId" />
      <IdentityComponent name="Host" useComputerName="true" />
    </Identity>
    <AgentResourceUsage diskQuotaInMB="50000" />
  </Management>

  <Schemas>
  </Schemas>

  <Sources>
  </Sources>

  <Events>
    <MdsdEvents>
    </MdsdEvents>

    <OMI>
    </OMI>

    <DerivedEvents>
    </DerivedEvents>
  </Events>

  <EventStreamingAnnotations>
  </EventStreamingAnnotations>

</MonitoringManagement>
"""


class LadConfigAll:
    """
    A class to generate configs for all 3 core components of LAD: mdsd, omsagent (fluentd), and syslog
    (rsyslog or syslog-ng) based on LAD's JSON extension settings.
    The mdsd XML config file generated will be /var/lib/waagent/Microsoft. ...-x.y.zzzz/xmlCfg.xml (hard-coded).
    Other config files whose contents are generated by this class are as follows:
    - /etc/opt/microsoft/omsagent/LAD/conf/omsagent.d/syslog.conf : fluentd's syslog source config
    - /etc/opt/microsoft/omsagent/LAD/conf/omsagent.d/tail.conf : fluentd's tail source config (fileLogs)
    - /etc/opt/microsoft/omsagent/LAD/conf/omsagent.d/z_out_mdsd.conf : fluentd's out_mdsd out plugin config
    - /etc/rsyslog.conf or /etc/rsyslog.d/95-omsagent.conf: rsyslog config for LAD's syslog settings
       The content should be appended to the corresponding file, not overwritten. After that, the file should be
       processed so that the '%SYSLOG_PORT%' pattern is replaced with the assigned TCP port number.
    - /etc/syslog-ng.conf: syslog-ng config for LAD's syslog settings. The content should be appended, not overwritten.
    """
    _default_perf_cfgs = [
        {"query": "SELECT PercentAvailableMemory, AvailableMemory, UsedMemory, PercentUsedSwap "
                  "FROM SCX_MemoryStatisticalInformation",
         "table": "LinuxMemory"},
        {"query": "SELECT PercentProcessorTime, PercentIOWaitTime, PercentIdleTime "
                  "FROM SCX_ProcessorStatisticalInformation WHERE Name='_TOTAL'",
         "table": "LinuxCpu"},
        {"query": "SELECT AverageWriteTime,AverageReadTime,ReadBytesPerSecond,WriteBytesPerSecond "
                  "FROM  SCX_DiskDriveStatisticalInformation WHERE Name='_TOTAL'",
         "table": "LinuxDisk"}
    ]

    def __init__(self, ext_settings, ext_dir, waagent_dir, deployment_id,
                 fetch_uuid, encrypt_string, logger_log, logger_error):
        """
        Constructor.
        :param ext_settings: A LadExtSettings (in Utils/lad_ext_settings.py) obj wrapping the Json extension settings.
        :param ext_dir: Extension directory (e.g., /var/lib/waagent/Microsoft.OSTCExtensions.LinuxDiagnostic-2.3.xxxx)
        :param waagent_dir: WAAgent directory (e.g., /var/lib/waagent)
        :param deployment_id: Deployment ID string (or None) that should be obtained & passed by the caller
                              from waagent's HostingEnvironmentCfg.xml.
        :param fetch_uuid: A function which fetches the UUID for the VM
        :param encrypt_string: A function which encrypts a string, given a cert_path
        :param logger_log: Normal logging function (e.g., hutil.log) that takes only one param for the logged msg.
        :param logger_error: Error logging function (e.g., hutil.error) that takes only one param for the logged msg.
        """
        self._ext_settings = ext_settings
        self._ext_dir = ext_dir
        self._waagent_dir = waagent_dir
        self._deployment_id = deployment_id
        self._fetch_uuid = fetch_uuid
        self._encrypt_secret = encrypt_string
        self._logger_log = logger_log
        self._logger_error = logger_error

        # Generated logging configs place holders
        self._fluentd_syslog_src_config = None
        self._fluentd_tail_src_config = None
        self._fluentd_out_mdsd_config = None
        self._rsyslog_config = None
        self._syslog_ng_config = None

        self._mdsd_config_xml_tree = ET.ElementTree(ET.fromstring(_mdsd_xml_template))
        self._sink_configs = LadUtil.SinkConfiguration()
        self._sink_configs.insert_from_config(self._ext_settings.read_protected_config('sinksConfig'))
        # If we decide to also read sinksConfig from ladCfg, do it first, so that private settings override

        # Get encryption settings
        thumbprint = ext_settings.get_handler_settings()['protectedSettingsCertThumbprint']
        path = '{0}/{1}.{2}'
        self._cert_path = path.format(waagent_dir, thumbprint, 'crt')
        self._pkey_path = path.format(waagent_dir, thumbprint, 'prv')

    def _ladCfg(self):
        return self._ext_settings.read_public_config('ladCfg')

    def _add_portal_settings(self, resource_id):
        """
        Update mdsd_config_xml_tree for Azure Portal metric collection setting.
        It's basically applying the resource_id as the partitionKey attribute of LADQuery elements.

        :param resource_id: ARM resource ID to provide as partitionKey in LADQuery elements
        :return: None
        """
        XmlUtil.setXmlValue(self._mdsd_config_xml_tree,
                            './DerivedEvents/DerivedEvent/LADQuery',
                            'partitionKey',
                            resource_id)

    def _update_metric_collection_settings(self, ladCfg):
        """
        Update mdsd_config_xml_tree for Azure Portal metric collection. The mdsdCfg performanceCounters element contains<commit_msg>Use os.path.join to combine paths with filenames<commit_after>#!/usr/bin/env python
#
# Azure Linux extension
#
# Linux Azure Diagnostic Extension (Current version is specified in manifest.xml)
# Copyright (c) Microsoft Corporation
# All rights reserved.
# MIT License
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
#  documentation files (the ""Software""), to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
#  permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
#  Software.
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
#  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
#  OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from misc_helpers import get_storage_endpoint_with_account, escape_nonalphanumerics
import os.path
import traceback
import xml.etree.ElementTree as ET
import Utils.ApplicationInsightsUtil as AIUtil
import Utils.LadDiagnosticUtil as LadUtil
import Utils.XmlUtil as XmlUtil
from Utils.lad_logging_config import LadLoggingConfig, copy_source_mdsdevent_elems, LadLoggingConfigException
import Providers.Builtin as BuiltIn


_mdsd_xml_template = """
<MonitoringManagement eventVersion="2" namespace="" timestamp="2017-03-27T19:45:00.000" version="1.0">
  <Accounts>
    <Account account="" isDefault="true" key="" moniker="moniker" tableEndpoint="" />
    <SharedAccessSignature account="" isDefault="true" key="" moniker="moniker" tableEndpoint="" />
  </Accounts>

  <Management defaultRetentionInDays="90" eventVolume="">
    <Identity>
      <IdentityComponent name="DeploymentId" />
      <IdentityComponent name="Host" useComputerName="true" />
    </Identity>
    <AgentResourceUsage diskQuotaInMB="50000" />
  </Management>

  <Schemas>
  </Schemas>

  <Sources>
  </Sources>

  <Events>
    <MdsdEvents>
    </MdsdEvents>

    <OMI>
    </OMI>

    <DerivedEvents>
    </DerivedEvents>
  </Events>

  <EventStreamingAnnotations>
  </EventStreamingAnnotations>

</MonitoringManagement>
"""


class LadConfigAll:
    """
    A class to generate configs for all 3 core components of LAD: mdsd, omsagent (fluentd), and syslog
    (rsyslog or syslog-ng) based on LAD's JSON extension settings.
    The mdsd XML config file generated will be /var/lib/waagent/Microsoft. ...-x.y.zzzz/xmlCfg.xml (hard-coded).
    Other config files whose contents are generated by this class are as follows:
    - /etc/opt/microsoft/omsagent/LAD/conf/omsagent.d/syslog.conf : fluentd's syslog source config
    - /etc/opt/microsoft/omsagent/LAD/conf/omsagent.d/tail.conf : fluentd's tail source config (fileLogs)
    - /etc/opt/microsoft/omsagent/LAD/conf/omsagent.d/z_out_mdsd.conf : fluentd's out_mdsd out plugin config
    - /etc/rsyslog.conf or /etc/rsyslog.d/95-omsagent.conf: rsyslog config for LAD's syslog settings
       The content should be appended to the corresponding file, not overwritten. After that, the file should be
       processed so that the '%SYSLOG_PORT%' pattern is replaced with the assigned TCP port number.
    - /etc/syslog-ng.conf: syslog-ng config for LAD's syslog settings. The content should be appended, not overwritten.
    """
    _default_perf_cfgs = [
        {"query": "SELECT PercentAvailableMemory, AvailableMemory, UsedMemory, PercentUsedSwap "
                  "FROM SCX_MemoryStatisticalInformation",
         "table": "LinuxMemory"},
        {"query": "SELECT PercentProcessorTime, PercentIOWaitTime, PercentIdleTime "
                  "FROM SCX_ProcessorStatisticalInformation WHERE Name='_TOTAL'",
         "table": "LinuxCpu"},
        {"query": "SELECT AverageWriteTime,AverageReadTime,ReadBytesPerSecond,WriteBytesPerSecond "
                  "FROM  SCX_DiskDriveStatisticalInformation WHERE Name='_TOTAL'",
         "table": "LinuxDisk"}
    ]

    def __init__(self, ext_settings, ext_dir, waagent_dir, deployment_id,
                 fetch_uuid, encrypt_string, logger_log, logger_error):
        """
        Constructor.
        :param ext_settings: A LadExtSettings (in Utils/lad_ext_settings.py) obj wrapping the Json extension settings.
        :param ext_dir: Extension directory (e.g., /var/lib/waagent/Microsoft.OSTCExtensions.LinuxDiagnostic-2.3.xxxx)
        :param waagent_dir: WAAgent directory (e.g., /var/lib/waagent)
        :param deployment_id: Deployment ID string (or None) that should be obtained & passed by the caller
                              from waagent's HostingEnvironmentCfg.xml.
        :param fetch_uuid: A function which fetches the UUID for the VM
        :param encrypt_string: A function which encrypts a string, given a cert_path
        :param logger_log: Normal logging function (e.g., hutil.log) that takes only one param for the logged msg.
        :param logger_error: Error logging function (e.g., hutil.error) that takes only one param for the logged msg.
        """
        self._ext_settings = ext_settings
        self._ext_dir = ext_dir
        self._waagent_dir = waagent_dir
        self._deployment_id = deployment_id
        self._fetch_uuid = fetch_uuid
        self._encrypt_secret = encrypt_string
        self._logger_log = logger_log
        self._logger_error = logger_error

        # Generated logging configs place holders
        self._fluentd_syslog_src_config = None
        self._fluentd_tail_src_config = None
        self._fluentd_out_mdsd_config = None
        self._rsyslog_config = None
        self._syslog_ng_config = None

        self._mdsd_config_xml_tree = ET.ElementTree(ET.fromstring(_mdsd_xml_template))
        self._sink_configs = LadUtil.SinkConfiguration()
        self._sink_configs.insert_from_config(self._ext_settings.read_protected_config('sinksConfig'))
        # If we decide to also read sinksConfig from ladCfg, do it first, so that private settings override

        # Get encryption settings
        thumbprint = ext_settings.get_handler_settings()['protectedSettingsCertThumbprint']
        path = '{0}/{1}.{2}'
        self._cert_path = os.path.join(waagent_dir, thumbprint + '.crt')
        self._pkey_path = os.path.join(waagent_dir, thumbprint + '.prv')

    def _ladCfg(self):
        return self._ext_settings.read_public_config('ladCfg')

    def _add_portal_settings(self, resource_id):
        """
        Update mdsd_config_xml_tree for Azure Portal metric collection setting.
        It's basically applying the resource_id as the partitionKey attribute of LADQuery elements.

        :param resource_id: ARM resource ID to provide as partitionKey in LADQuery elements
        :return: None
        """
        XmlUtil.setXmlValue(self._mdsd_config_xml_tree,
                            './DerivedEvents/DerivedEvent/LADQuery',
                            'partitionKey',
                            resource_id)

    def _update_metric_collection_settings(self, ladCfg):
        """
        Update mdsd_config_xml_tree for Azure Portal metric collection. The mdsdCfg performanceCounters element contains
{'fraction_non_alphanumeric': 0.08160561701000199, 'fraction_numerical': 0.0046366827846592035, 'mean_word_length': 3.4681858538029005, 'pattern_counts': {'":': 12, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7458687', 'n_tokens_mistral': 4379, 'n_tokens_neox': 4200, 'n_words': 1321}