<commit_before>#!/usr/bin/env python

'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

def fix_encoding_reimport_bug():
  """
  Fix https://bugs.python.org/issue14847
  """
  b'x'.decode('utf-8')
  b'x'.decode('ascii')

def fix_subprocess_racecondition():
  """
  Subprocess in Python has race condition with enabling/disabling gc. Which may lead to turning off python garbage collector.
  This leads to a memory leak.
  This function monkey patches subprocess to fix the issue.

  !!! PLEASE NOTE THIS SHOULD BE CALLED BEFORE ANY OTHER INITIALIZATION was done to avoid already created links to subprocess or subprocess.gc or gc
  """
  # monkey patching subprocess
  import subprocess
  subprocess.gc.isenabled = lambda: True

  # re-importing gc to have correct isenabled for non-subprocess contexts
  import sys
  del sys.modules['gc']
  import gc


def fix_subprocess_popen():
  '''
  Workaround for race condition in starting subprocesses concurrently from
  multiple threads via the subprocess and multiprocessing modules.
  See http://bugs.python.org/issue19809 for details and repro script.
  '''
  import os
  import sys

  if os.name == 'posix' and sys.version_info[0] < 3:
    from multiprocessing import forking
    import subprocess
    import threading

    sp_original_init = subprocess.Popen.__init__
    mp_original_init = forking.Popen.__init__
    lock = threading.RLock() # guards subprocess creation

    def sp_locked_init(self, *a, **kw):
      with lock:
        sp_original_init(self, *a, **kw)

    def mp_locked_init(self, *a, **kw):
      with lock:
        mp_original_init(self, *a, **kw)

    subprocess.Popen.__init__ = sp_locked_init
    forking.Popen.__init__ = mp_locked_init


fix_subprocess_popen()
fix_subprocess_racecondition()
fix_encoding_reimport_bug()

import logging.handlers
import logging.config
import signal
from optparse import OptionParser
import sys
import traceback
import getpass
import os
import time
import locale
import platform
import ConfigParser
import ProcessHelper
from logging.handlers import SysLogHandler
from Controller import Controller
import AmbariConfig
from NetUtil import NetUtil
from PingPortListener import PingPortListener
import hostname
from DataCleaner import DataCleaner
from ambari_agent.ExitHelper import ExitHelper
import socket
from ambari_commons import OSConst, OSCheck
from ambari_commons.shell import shellRunner
from ambari_commons import shell
import HeartbeatHandlers
from HeartbeatHandlers import bind_signal_handlers
from ambari_commons.constants import AMBARI_SUDO_BINARY
from resource_management.core.logger import Logger
logger = logging.getLogger()
alerts_logger = logging.getLogger('ambari_alerts')

formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
agentPid = os.getpid()

# Global variables to be set later.
home_dir = ""

config = AmbariConfig.AmbariConfig()
# TODO AMBARI-18733, remove this global variable and calculate it based on home_dir once it is set.
configFile = config.getConfigFile()
two_way_ssl_property = config.TWO_WAY_SSL_PROPERTY

IS_LINUX = platform.system() == "Linux"
SYSLOG_FORMAT_STRING = ' ambari_agent - %(filename)s - [%(process)d] - %(name)s - %(levelname)s - %(message)s'
SYSLOG_FORMATTER = logging.Formatter(SYSLOG_FORMAT_STRING)

def setup_logging(logger, filename, logging_level):
  formatter = logging.Formatter(formatstr)
  rotateLog = logging.handlers.RotatingFileHandler(filename, "a", 10000000, 25)
  rotateLog.setFormatter(formatter)
  logger.addHandler(rotateLog)
      
  logging.basicConfig(format=formatstr, level=logging_level, filename=filename)
  logger.setLevel(logging_level)
  logger.info("loglevel=logging.{0}".format(logging._levelNames[logging_level]))

GRACEFUL_STOP_TRIES = 10<commit_msg>AMBARI-20504. Ambari-agent log rotation is broken (echekanskiy)<commit_after>#!/usr/bin/env python

'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

def fix_encoding_reimport_bug():
  """
  Fix https://bugs.python.org/issue14847
  """
  b'x'.decode('utf-8')
  b'x'.decode('ascii')

def fix_subprocess_racecondition():
  """
  Subprocess in Python has race condition with enabling/disabling gc. Which may lead to turning off python garbage collector.
  This leads to a memory leak.
  This function monkey patches subprocess to fix the issue.

  !!! PLEASE NOTE THIS SHOULD BE CALLED BEFORE ANY OTHER INITIALIZATION was done to avoid already created links to subprocess or subprocess.gc or gc
  """
  # monkey patching subprocess
  import subprocess
  subprocess.gc.isenabled = lambda: True

  # re-importing gc to have correct isenabled for non-subprocess contexts
  import sys
  del sys.modules['gc']
  import gc


def fix_subprocess_popen():
  '''
  Workaround for race condition in starting subprocesses concurrently from
  multiple threads via the subprocess and multiprocessing modules.
  See http://bugs.python.org/issue19809 for details and repro script.
  '''
  import os
  import sys

  if os.name == 'posix' and sys.version_info[0] < 3:
    from multiprocessing import forking
    import subprocess
    import threading

    sp_original_init = subprocess.Popen.__init__
    mp_original_init = forking.Popen.__init__
    lock = threading.RLock() # guards subprocess creation

    def sp_locked_init(self, *a, **kw):
      with lock:
        sp_original_init(self, *a, **kw)

    def mp_locked_init(self, *a, **kw):
      with lock:
        mp_original_init(self, *a, **kw)

    subprocess.Popen.__init__ = sp_locked_init
    forking.Popen.__init__ = mp_locked_init


fix_subprocess_popen()
fix_subprocess_racecondition()
fix_encoding_reimport_bug()

import logging.handlers
import logging.config
import signal
from optparse import OptionParser
import sys
import traceback
import getpass
import os
import time
import locale
import platform
import ConfigParser
import ProcessHelper
from logging.handlers import SysLogHandler
from Controller import Controller
import AmbariConfig
from NetUtil import NetUtil
from PingPortListener import PingPortListener
import hostname
from DataCleaner import DataCleaner
from ambari_agent.ExitHelper import ExitHelper
import socket
from ambari_commons import OSConst, OSCheck
from ambari_commons.shell import shellRunner
from ambari_commons import shell
import HeartbeatHandlers
from HeartbeatHandlers import bind_signal_handlers
from ambari_commons.constants import AMBARI_SUDO_BINARY
from resource_management.core.logger import Logger

logger = logging.getLogger()
alerts_logger = logging.getLogger('ambari_alerts')

formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
agentPid = os.getpid()

# Global variables to be set later.
home_dir = ""

config = AmbariConfig.AmbariConfig()
# TODO AMBARI-18733, remove this global variable and calculate it based on home_dir once it is set.
configFile = config.getConfigFile()
two_way_ssl_property = config.TWO_WAY_SSL_PROPERTY

IS_LINUX = platform.system() == "Linux"
SYSLOG_FORMAT_STRING = ' ambari_agent - %(filename)s - [%(process)d] - %(name)s - %(levelname)s - %(message)s'
SYSLOG_FORMATTER = logging.Formatter(SYSLOG_FORMAT_STRING)

_file_logging_handlers ={}

def setup_logging(logger, filename, logging_level):
  formatter = logging.Formatter(formatstr)

  if filename in _file_logging_handlers:
    rotateLog = _file_logging_handlers[filename]
  else:
    rotateLog = logging.handlers.RotatingFileHandler(filename, "a", 10000000, 25)
    rotateLog.setFormatter(formatter)
    _file_logging_handlers[filename] = rotateLog
  logger.addHandler(rotateLog)
      
  logging.basicConfig(format=formatstr, level=logging_level, filename=filename)
  logger.setLevel(logging_level)
  logger.info("loglevel=logging.{0}".format(logging._levelNames[logging_level]))

GRACEFUL_STOP_TRIES = 10
{'fraction_non_alphanumeric': 0.07483955183291635, 'fraction_numerical': 0.008158381377134776, 'mean_word_length': 5.032808398950131, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6640083', 'n_tokens_mistral': 2813, 'n_tokens_neox': 2670, 'n_words': 887}