<commit_before>#!/usr/bin/python
#
# Copyright (c) 2008 rPath, Inc.
#

import os
import time

from amiconfig.lib import util
from amiconfig.lib.daemon import Daemon

class SpaceAllocator(object):
    def __init__(self, path):
        self._path = path
        self._allocated = 0

    def _getFileSystemUsage(self):
        """ Returns disk usage on self._path in MB """
        fs = os.statvfs(self._path)
        blocksUsed = fs.f_blocks - fs.f_bfree
        sizeUsed = blocksUsed * fs.f_bsize / 1024 / 1024
        return sizeUsed

    def grow(self, size):
        # size is in MB
        # make sure that 20% over current usage is preallocated
        fh = util.createUnlinkedTmpFile(self._path)
        while self._allocated < 1.2 * self._getFileSystemUsage():
            util.growFile(fh, size)
            self._allocated += size
        fh.close()


class SpaceDaemon(Daemon):
    def __init__(self, paths, size):
        self._paths = paths
        self._size = int(size)
        self._fs = []
        for path in self._paths:
            self._fs.append(SpaceAllocator(path))

    def start(self):
        """ Every 60s try to allocate more space. """
        while True:
            for fs in self._fs:
                fs.grow(self._size)
            time.sleep(60)


if __name__ == '__main__':
    import sys

    if len(sys.argv) < 3:
        print "usage: %s <size in MB> <path, path, ...>" % sys.argv[0]
        sys.exit(1)

    size = sys.argv[1]
    paths = sys.argv[2:]

    d = SpaceDaemon(paths, size)
    sys.exit(d.daemonize())<commit_msg>1. Add wait before trying to allocate space the first time so that the machine can finishing booting first. 2. Don't allocate space as often, this should reduce some load on the machine.<commit_after>#!/usr/bin/python
#
# Copyright (c) 2008 rPath, Inc.
#

import os
import time

from amiconfig.lib import util
from amiconfig.lib.daemon import Daemon

class SpaceAllocator(object):
    def __init__(self, path):
        self._path = path
        self._allocated = 0

    def _getFileSystemUsage(self):
        """ Returns disk usage on self._path in MB """
        fs = os.statvfs(self._path)
        blocksUsed = fs.f_blocks - fs.f_bfree
        sizeUsed = blocksUsed * fs.f_bsize / 1024 / 1024
        return sizeUsed

    def grow(self, size):
        # size is in MB
        # make sure that 20% over current usage is preallocated
        fh = util.createUnlinkedTmpFile(self._path)
        while self._allocated < 1.2 * self._getFileSystemUsage():
            util.growFile(fh, size)
            self._allocated += size
        fh.close()


class SpaceDaemon(Daemon):
    def __init__(self, paths, size):
        self._paths = paths
        self._size = int(size)
        self._fs = []
        for path in self._paths:
            self._fs.append(SpaceAllocator(path))

    def start(self):
        """ Every 5min try to allocate more space. """
        # Wait a couple of minutes before starting allocation to let the
        # machine finish booting.
        time.sleep(120)
        while True:
            for fs in self._fs:
                fs.grow(self._size)
            time.sleep(300)


if __name__ == '__main__':
    import sys

    if len(sys.argv) < 3:
        print "usage: %s <size in MB> <path, path, ...>" % sys.argv[0]
        sys.exit(1)

    size = sys.argv[1]
    paths = sys.argv[2:]

    d = SpaceDaemon(paths, size)
    sys.exit(d.daemonize())
{'fraction_non_alphanumeric': 0.10722610722610723, 'fraction_numerical': 0.016608391608391608, 'mean_word_length': 2.1437728937728937, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5740977', 'n_tokens_mistral': 1127, 'n_tokens_neox': 1054, 'n_words': 314}