<commit_before># dblite.py module contributed by <NAME>.
# Extended for Unicode by <NAME>.

import cPickle
import time
import shutil
import os
import types
import __builtin__

keep_all_files = 00000
ignore_corrupt_dbfiles = 0

def corruption_warning(filename):
    print "Warning: Discarding corrupt database:", filename

if hasattr(types, 'UnicodeType'):
    def is_string(s):
        t = type(s)
        return t is types.StringType or t is types.UnicodeType
else:
    def is_string(s):
        return type(s) is types.StringType

try:
    unicode('a')
except NameError:
    def unicode(s): return s

dblite_suffix = '.dblite'
tmp_suffix = '.tmp'

class dblite:

  # Squirrel away references to the functions in various modules
  # that we'll use when our __del__() method calls our sync() method
  # during shutdown.  We might get destroyed when Python is in the midst
  # of tearing down the different modules we import in an essentially
  # arbitrary order, and some of the various modules's global attributes
  # may already be wiped out from under us.
  #
  # See the discussion at:
  #   http://mail.python.org/pipermail/python-bugs-list/2003-March/016877.html

  _open = __builtin__.open
  _cPickle_dump = cPickle.dump
  _os_chmod = os.chmod
  _os_rename = os.rename
  _os_unlink = os.unlink
  _shutil_copyfile = shutil.copyfile
  _time_time = time.time

  def __init__(self, file_base_name, flag, mode):
    assert flag in (None, "r", "w", "c", "n")
    if (flag is None): flag = "r"
    base, ext = os.path.splitext(file_base_name)
    if ext == dblite_suffix:
      # There's already a suffix on the file name, don't add one.
      self._file_name = file_base_name
      self._tmp_name = base + tmp_suffix
    else:
      self._file_name = file_base_name + dblite_suffix
      self._tmp_name = file_base_name + tmp_suffix
    self._flag = flag
    self._mode = mode
    self._dict = {}
    self._needs_sync = 00000
    if (self._flag == "n"):
      self._open(self._file_name, "wb", self._mode)
    else:
      try:
        f = self._open(self._file_name, "rb")
      except IOError, e:
        if (self._flag != "c"):
          raise e
        self._open(self._file_name, "wb", self._mode)
      else:
        p = f.read()
        if (len(p) > 0):
          try:
            self._dict = cPickle.loads(p)
          except (cPickle.UnpicklingError, EOFError):
            if (ignore_corrupt_dbfiles == 0): raise
            if (ignore_corrupt_dbfiles == 1):
              corruption_warning(self._file_name)

  def __del__(self):
    if (self._needs_sync):
      self.sync()

  def sync(self):
    self._check_writable()
    f = self._open(self._tmp_name, "wb", self._mode)
    self._cPickle_dump(self._dict, f, 1)
    f.close()
    # Windows doesn't allow renaming if the file exists, so unlink
    # it first, chmod'ing it to make sure we can do so.  On UNIX, we
    # may not be able to chmod the file if it's owned by someone else
    # (e.g. from a previous run as root).  We should still be able to
    # unlink() the file if the directory's writable, though, so ignore
    # any OSError exception  thrown by the chmod() call.
    try: self._os_chmod(self._file_name, 0777)
    except OSError: pass
    self._os_unlink(self._file_name)
    self._os_rename(self._tmp_name, self._file_name)
    self._needs_sync = 00000
    if (keep_all_files):
      self._shutil_copyfile(
        self._file_name,
        self._file_name + "_" + str(int(self._time_time())))

  def _check_writable(self):
    if (self._flag == "r"):
      raise IOError("Read-only database: %s" % self._file_name)

  def __getitem__(self, key):
    return self._dict[key]

  def __setitem__(self, key, value):
    self._check_writable()
    if (not is_string(key)):
      raise TypeError, "key `%s' must be a string but is %s" % (key, type(key))
    if (not is_string(value)):
      raise TypeError, "value `%s' must be a string but is %s" % (value, type(value))
    self._dict[key] = value<commit_msg>Fix issue 1382: chown .sconsign.dblite back to original user/group when done, when running as root.  If it didn't exist, chown to user who executed sudo if known.  Hard to test automatically because test needs to run as root.<commit_after># dblite.py module contributed by <NAME>.
# Extended for Unicode by <NAME>.

import cPickle
import time
import shutil
import os
import types
import __builtin__

keep_all_files = 00000
ignore_corrupt_dbfiles = 0

def corruption_warning(filename):
    print "Warning: Discarding corrupt database:", filename

if hasattr(types, 'UnicodeType'):
    def is_string(s):
        t = type(s)
        return t is types.StringType or t is types.UnicodeType
else:
    def is_string(s):
        return type(s) is types.StringType

try:
    unicode('a')
except NameError:
    def unicode(s): return s

dblite_suffix = '.dblite'
tmp_suffix = '.tmp'

class dblite:

  # Squirrel away references to the functions in various modules
  # that we'll use when our __del__() method calls our sync() method
  # during shutdown.  We might get destroyed when Python is in the midst
  # of tearing down the different modules we import in an essentially
  # arbitrary order, and some of the various modules's global attributes
  # may already be wiped out from under us.
  #
  # See the discussion at:
  #   http://mail.python.org/pipermail/python-bugs-list/2003-March/016877.html

  _open = __builtin__.open
  _cPickle_dump = cPickle.dump
  _os_chmod = os.chmod
  _os_chown = os.chown
  _os_rename = os.rename
  _os_unlink = os.unlink
  _shutil_copyfile = shutil.copyfile
  _time_time = time.time

  def __init__(self, file_base_name, flag, mode):
    assert flag in (None, "r", "w", "c", "n")
    if (flag is None): flag = "r"
    base, ext = os.path.splitext(file_base_name)
    if ext == dblite_suffix:
      # There's already a suffix on the file name, don't add one.
      self._file_name = file_base_name
      self._tmp_name = base + tmp_suffix
    else:
      self._file_name = file_base_name + dblite_suffix
      self._tmp_name = file_base_name + tmp_suffix
    self._flag = flag
    self._mode = mode
    self._dict = {}
    self._needs_sync = 00000
    if os.geteuid()==0 or os.getuid()==0:
      # running as root; chown back to current owner/group when done
      try:
        statinfo = os.stat(self._file_name)
        self._chown_to = statinfo.st_uid
        self._chgrp_to = statinfo.st_gid
      except OSError, e:
        # db file doesn't exist yet.
        # Check os.environ for SUDO_UID, use if set
        self._chown_to = int(os.environ.get('SUDO_UID', -1))
        self._chgrp_to = int(os.environ.get('SUDO_GID', -1))
    else:
      self._chown_to = -1        # don't chown
      self._chgrp_to = -1        # don't chgrp
    if (self._flag == "n"):
      self._open(self._file_name, "wb", self._mode)
    else:
      try:
        f = self._open(self._file_name, "rb")
      except IOError, e:
        if (self._flag != "c"):
          raise e
        self._open(self._file_name, "wb", self._mode)
      else:
        p = f.read()
        if (len(p) > 0):
          try:
            self._dict = cPickle.loads(p)
          except (cPickle.UnpicklingError, EOFError):
            if (ignore_corrupt_dbfiles == 0): raise
            if (ignore_corrupt_dbfiles == 1):
              corruption_warning(self._file_name)

  def __del__(self):
    if (self._needs_sync):
      self.sync()

  def sync(self):
    self._check_writable()
    f = self._open(self._tmp_name, "wb", self._mode)
    self._cPickle_dump(self._dict, f, 1)
    f.close()
    # Windows doesn't allow renaming if the file exists, so unlink
    # it first, chmod'ing it to make sure we can do so.  On UNIX, we
    # may not be able to chmod the file if it's owned by someone else
    # (e.g. from a previous run as root).  We should still be able to
    # unlink() the file if the directory's writable, though, so ignore
    # any OSError exception  thrown by the chmod() call.
    try: self._os_chmod(self._file_name, 0777)
    except OSError: pass
    self._os_unlink(self._file_name)
    self._os_rename(self._tmp_name, self._file_name)
    if self._chown_to > 0:      # don't chown to root or -1
      try:
        self._os_chown(self._file_name, self._chown_to, self._chgrp_to)
      except OSError:
        pass
    self._needs_sync = 00000
    if (keep_all_files):
      self._shutil_copyfile(
        self._file_name,
        self._file_name + "_" + str(int(self._time_time())))

  def _check_writable(self):
    if (self._flag == "r"):
      raise IOError("Read-only database: %s" % self._file_name)

  def __getitem__(self, key):
    return self._dict[key]

  def __setitem__(self, key, value):
    self._check_writable()
    if (not is_string(key)):
      raise TypeError, "key `%s' must be a string but is %s" % (key, type(key))
    if (not is_string(value)):
      raise TypeError, "value `%s' must be a string but is %s" % (value, type(value))
    self._dict[key] = value
{'fraction_non_alphanumeric': 0.10553883873843753, 'fraction_numerical': 0.008915635796277722, 'mean_word_length': 2.833404527979496, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4091246', 'n_tokens_mistral': 3147, 'n_tokens_neox': 3032, 'n_words': 913}