<commit_before># Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A class for managing the Linux cgroup subsystem."""

import contextlib
import errno
import os
import signal
import time

from chromite.lib import cros_build_lib
from chromite.lib import locking
from chromite.lib import osutils
from chromite.lib import signals
from chromite.lib import sudo


# Rough hierarchy sketch:
# - all cgroup aware cros code should nest here.
# - No cros code should modify this namespace- this is user/system configurable
# - only.  A release_agent can be specified, although we won't use it.
# cros/
#
# - cbuildbot instances land here only when they're cleaning their task pool.
# - this root namespace is *not* auto-removed; it's left so that user/system
# - configuration is persistant.
# cros/%(process-name)s/
# cros/cbuildbot/
#
# - a cbuildbot job pool, owned by pid.  These are autocleaned.
# cros/cbuildbot/%(pid)i/
#
# - a job pool using process that was invoked by cbuildbot.
# - for example, cros/cbuildbot/42/cros_sdk:34
# - this pattern continues arbitrarily deep, and is autocleaned.
# cros/cbuildbot/%(pid1)i/%(basename_of_pid2)s:%(pid2)i/
#
# An example for cros_sdk (pid 552) would be:
# cros/cros_sdk/552/
# and it's children would be accessible in 552/tasks, or
# would create their own namespace w/in and assign themselves to it.


class _GroupWasRemoved(Exception):
  """Exception representing when a group was unexpectedly removed.

  Via design, this should only be possible when instantiating a new
  pool, but the parent pool has been removed- this means effectively that
  we're supposed to shutdown (either we've been sigterm'd and ignored it,
  or it's imminent)."""


def _FileContains(filename, strings):
  """Greps a group of expressions, returns whether all were found."""
  contents = osutils.ReadFile(filename)
  return all(s in contents for s in strings)


def MemoizedSingleCall(functor):
  """Decorator for simple functor targets, caching the results

  The functor must accept no arguments beyond either a class or self (depending
  on if this is used in a classmethod/instancemethod context).  Results of the
  wrapped method will be written to the class/instance namespace in a specially
  named cached value.  All future invocations will just reuse that value."""
  # TODO(ferringb): rebase to snakeoil.klass.cached* functionality if/when
  # snakeoil occurs.
  def f(obj):
    # Silence idiotic complaint from pylint.
    # pylint: disable=W0212
    key = f._cache_key
    val = getattr(obj, key, None)
    if val is None:
      val = functor(obj)
      setattr(obj, key, val)
    return val

  # Dummy up our wrapper to make it look like what we're wrapping,
  # and expose the underlying docstrings.
  f.__name__ = functor.__name__
  f.__module__ = functor.__module__
  f.__doc__ = functor.__doc__
  f._cache_key = '_%s_cached' % (functor.__name__.lstrip('_'),)
  return f


def EnsureInitialized(functor):
  """Decorator for Cgroup methods to ensure the method is ran only if inited"""

  def f(self, *args, **kwds):
    # pylint: disable=W0212
    self.Instantiate()
    return functor(self, *args, **kwds)

  # Dummy up our wrapper to make it look like what we're wrapping,
  # and expose the underlying docstrings.
  f.__name__ = functor.__name__
  f.__doc__ = functor.__doc__
  f.__module__ = functor.__module__
  return f


class Cgroup(object):

  """Class representing a group in cgroups hierarchy.

  Note the instance may not exist on disk; it will be created as necessary.
  Additionally, because cgroups is kernel maintained (and mutated on the fly
  by processes using it), chunks of this class are /explicitly/ designed to
  always go back to disk and recalculate values.

  Attributes:
    path: Absolute on disk pathway to the cgroup directory.
    tasks: Pids contained in this immediate cgroup, and the owning pids of
      any first level groups nested w/in us.
    all_tasks: All Pids, and owners of nested groups w/in this point in
      the hierarchy.
    nested_groups: The immediate cgroups nested w/in this one.  If this
      cgroup is 'cbuildbot/buildbot', 'cbuildbot' would have a nested_groups
      of [Cgroup('cbuildbot/buildbot')] for example.
    all_nested_groups: All cgroups nested w/in this one, regardless of depth.
    pid_owner: Which pid owns this cgroup, if the cgroup is following cros
      conventions for group naming.
  """

  NEEDED_SUBSYSTEMS = ('cpuset',)
  PROC_PATH = '/proc/cgroups'
  _MOUNT_ROOT_POTENTIALS = ('/sys/fs/cgroup',)
  _MOUNT_ROOT_FALLBACK = '/dev/cgroup'
  CGROUP_ROOT = None
  MOUNT_ROOT = None
  # Whether or not the cgroup implementation does auto inheritance via
  # cgroup.clone_children
  _SUPPORTS_AUTOINHERIT = False

  @classmethod
  @MemoizedSingleCall
  def InitSystem(cls):
    """If cgroups are supported, initialize the system state"""
    if not cls.IsSupported():
      return False

    def _EnsureMounted(mnt, args):
      if _FileContains('/proc/mounts', [mnt]):
        return True

      # Grab a lock so in the off chance we have multiple programs (like two
      # cros_sdk launched in parallel) running this init logic, we don't end
      # up mounting multiple times.
      lock_path = '/tmp/.chromite.cgroups.lock'
      with locking.FileLock(lock_path, 'cgroup lock') as lock:
        lock.write_lock()
        if _FileContains('/proc/mounts', [mnt]):
          return True

        # Not all distros mount cgroup_root to sysfs.
        osutils.SafeMakedirs(mnt, sudo=True)
        cros_build_lib.SudoRunCommand(['mount'] + args + [mnt], print_cmd=False)

      return True

    mount_root_args = ['-t', 'tmpfs', 'cgroup_root']

    opts = ','.join(cls.NEEDED_SUBSYSTEMS)
    cgroup_root_args = ['-t', 'cgroup', '-o', opts, 'cros']

    return _EnsureMounted(cls.MOUNT_ROOT, mount_root_args) and \
        _EnsureMounted(cls.CGROUP_ROOT, cgroup_root_args)
<commit_msg>fix(cros_sdk): Hack around systemd's new restriction on cgroups.<commit_after># Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A class for managing the Linux cgroup subsystem."""

import contextlib
import errno
import os
import signal
import time

from chromite.lib import cros_build_lib
from chromite.lib import locking
from chromite.lib import osutils
from chromite.lib import signals
from chromite.lib import sudo


# Rough hierarchy sketch:
# - all cgroup aware cros code should nest here.
# - No cros code should modify this namespace- this is user/system configurable
# - only.  A release_agent can be specified, although we won't use it.
# cros/
#
# - cbuildbot instances land here only when they're cleaning their task pool.
# - this root namespace is *not* auto-removed; it's left so that user/system
# - configuration is persistant.
# cros/%(process-name)s/
# cros/cbuildbot/
#
# - a cbuildbot job pool, owned by pid.  These are autocleaned.
# cros/cbuildbot/%(pid)i/
#
# - a job pool using process that was invoked by cbuildbot.
# - for example, cros/cbuildbot/42/cros_sdk:34
# - this pattern continues arbitrarily deep, and is autocleaned.
# cros/cbuildbot/%(pid1)i/%(basename_of_pid2)s:%(pid2)i/
#
# An example for cros_sdk (pid 552) would be:
# cros/cros_sdk/552/
# and it's children would be accessible in 552/tasks, or
# would create their own namespace w/in and assign themselves to it.


class _GroupWasRemoved(Exception):
  """Exception representing when a group was unexpectedly removed.

  Via design, this should only be possible when instantiating a new
  pool, but the parent pool has been removed- this means effectively that
  we're supposed to shutdown (either we've been sigterm'd and ignored it,
  or it's imminent)."""


def _FileContains(filename, strings):
  """Greps a group of expressions, returns whether all were found."""
  contents = osutils.ReadFile(filename)
  return all(s in contents for s in strings)


def MemoizedSingleCall(functor):
  """Decorator for simple functor targets, caching the results

  The functor must accept no arguments beyond either a class or self (depending
  on if this is used in a classmethod/instancemethod context).  Results of the
  wrapped method will be written to the class/instance namespace in a specially
  named cached value.  All future invocations will just reuse that value."""
  # TODO(ferringb): rebase to snakeoil.klass.cached* functionality if/when
  # snakeoil occurs.
  def f(obj):
    # Silence idiotic complaint from pylint.
    # pylint: disable=W0212
    key = f._cache_key
    val = getattr(obj, key, None)
    if val is None:
      val = functor(obj)
      setattr(obj, key, val)
    return val

  # Dummy up our wrapper to make it look like what we're wrapping,
  # and expose the underlying docstrings.
  f.__name__ = functor.__name__
  f.__module__ = functor.__module__
  f.__doc__ = functor.__doc__
  f._cache_key = '_%s_cached' % (functor.__name__.lstrip('_'),)
  return f


def EnsureInitialized(functor):
  """Decorator for Cgroup methods to ensure the method is ran only if inited"""

  def f(self, *args, **kwds):
    # pylint: disable=W0212
    self.Instantiate()
    return functor(self, *args, **kwds)

  # Dummy up our wrapper to make it look like what we're wrapping,
  # and expose the underlying docstrings.
  f.__name__ = functor.__name__
  f.__doc__ = functor.__doc__
  f.__module__ = functor.__module__
  return f


class Cgroup(object):

  """Class representing a group in cgroups hierarchy.

  Note the instance may not exist on disk; it will be created as necessary.
  Additionally, because cgroups is kernel maintained (and mutated on the fly
  by processes using it), chunks of this class are /explicitly/ designed to
  always go back to disk and recalculate values.

  Attributes:
    path: Absolute on disk pathway to the cgroup directory.
    tasks: Pids contained in this immediate cgroup, and the owning pids of
      any first level groups nested w/in us.
    all_tasks: All Pids, and owners of nested groups w/in this point in
      the hierarchy.
    nested_groups: The immediate cgroups nested w/in this one.  If this
      cgroup is 'cbuildbot/buildbot', 'cbuildbot' would have a nested_groups
      of [Cgroup('cbuildbot/buildbot')] for example.
    all_nested_groups: All cgroups nested w/in this one, regardless of depth.
    pid_owner: Which pid owns this cgroup, if the cgroup is following cros
      conventions for group naming.
  """

  NEEDED_SUBSYSTEMS = ('cpuset',)
  PROC_PATH = '/proc/cgroups'
  _MOUNT_ROOT_POTENTIALS = ('/sys/fs/cgroup',)
  _MOUNT_ROOT_FALLBACK = '/dev/cgroup'
  CGROUP_ROOT = None
  MOUNT_ROOT = None
  # Whether or not the cgroup implementation does auto inheritance via
  # cgroup.clone_children
  _SUPPORTS_AUTOINHERIT = False

  @classmethod
  @MemoizedSingleCall
  def InitSystem(cls):
    """If cgroups are supported, initialize the system state"""
    if not cls.IsSupported():
      return False

    def _IsReadOnly(mnt):
      # from sys/mount.h
      MS_RDONLY = 1
      return os.statvfs(mnt).f_flag & MS_RDONLY

    def _EnsureMounted(mnt, args):
      # Grab a lock so in the off chance we have multiple programs (like two
      # cros_sdk launched in parallel) running this init logic, we don't end
      # up mounting multiple times.
      lock_path = '/tmp/.chromite.cgroups.lock'
      with locking.FileLock(lock_path, 'cgroup lock') as lock:
        lock.write_lock()
        if _FileContains('/proc/mounts', [mnt]):
          if _IsReadOnly(mnt):
            cros_build_lib.SudoRunCommand(['mount', '-o', 'remount,rw', mnt], print_cmd=False)
          return True

        # Not all distros mount cgroup_root to sysfs.
        osutils.SafeMakedirs(mnt, sudo=True)
        cros_build_lib.SudoRunCommand(['mount'] + args + [mnt], print_cmd=False)

      return True

    mount_root_args = ['-t', 'tmpfs', 'cgroup_root']

    opts = ','.join(cls.NEEDED_SUBSYSTEMS)
    cgroup_root_args = ['-t', 'cgroup', '-o', opts, 'cros']

    return _EnsureMounted(cls.MOUNT_ROOT, mount_root_args) and \
        _EnsureMounted(cls.CGROUP_ROOT, cgroup_root_args)

{'fraction_non_alphanumeric': 0.08458968299242116, 'fraction_numerical': 0.004645098199005786, 'mean_word_length': 3.6152688980819856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4351611', 'n_tokens_mistral': 3888, 'n_tokens_neox': 3681, 'n_words': 1424}