<commit_before>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import os
import re
import stat
import tempfile

from conary.lib import util
from conary.build import policy


class NormalizeCompression(policy.DestdirPolicy):
    """
    NAME
    ====

    B{C{r.NormalizeCompression()}} - Compress files with maximum compression

    SYNOPSIS
    ========

    C{r.NormalizeCompression([I{filterexp}] I{exceptions=filterexp}])}

    DESCRIPTION
    ===========

    The C{r.NormalizeCompression()} policy compresses files with maximum
    compression, and without data which may change from invocation, to
    invocation.

    Recompresses .gz files with -9 -n, and .bz2 files with -9, to get maximum
    compression and avoid meaningless changes overpopulating the database.
    Ignores man/info pages, as they are encountered separately while making other
    changes to man/info pages later.

    EXAMPLES
    ========

    C{r.NormalizeCompression(exceptions='%(thistestdir)s/.*')}

    This package has test files that are tested byte-for-byte and
    cannot be modified at all and still pass the tests.
    """
    invariantexceptions = [
        '%(mandir)s/man.*/',
        '%(infodir)s/',
    ]
    invariantinclusions = [
        ('.*\.(gz|bz2)', None, stat.S_IFDIR),
    ]
    def doFile(self, path):
        m = self.recipe.magic[path]
        if not m:
            return

        # Note: uses external gzip/bunzip if they exist because a
        # pipeline is faster in a multiprocessing environment

        def _mktmp(fullpath):
            fd, path = tempfile.mkstemp('.temp', '', os.path.dirname(fullpath))
            os.close(fd)
            return path

        def _move(tmppath, fullpath):
            os.chmod(tmppath, os.lstat(fullpath).st_mode)
            os.rename(tmppath, fullpath)

        fullpath = self.macros.destdir+path
        if m.name == 'gzip' and \
           (m.contents['compression'] != '9' or 'name' in m.contents):
            tmppath = _mktmp(fullpath)
            util.execute('/bin/gzip -dc %s | /bin/gzip -f -n -9 > %s'
                         %(fullpath, tmppath))
            _move(tmppath, fullpath)
            del self.recipe.magic[path]
        if m.name == 'bzip' and m.contents['compression'] != '9':
            tmppath = _mktmp(fullpath)
            util.execute('/usr/bin/bzip2 -dc %s | /usr/bin/bzip2 -9 > %s'
                         %(fullpath, tmppath))
            _move(tmppath, fullpath)
            del self.recipe.magic[path]


class NormalizeManPages(policy.DestdirPolicy):
    """
    NAME
    ====

    B{C{r.RemoveNonPackageFiles()}} - Make all man pages follow sane system policy

    SYNOPSIS
    ========

    C{r.RemoveNonPackageFiles([I{filterexp}] I{exceptions=filterexp}])}

    DESCRIPTION
    ===========

    The C{r.RemoveNonPackageFiles()} policy makes all system manual pages
    follow sane system policy

    Note: This policy class is not called directly from recipes, and does not
    honor exceptions.

    Some of the following tasks are performed against system manual pages via
    C{r.NormalizeManPages}:

    - Fix all man pages' contents:
    - remove instances of C{/?%(destdir)s} from all man pages
    - C{.so foo.n} becomes a symlink to foo.n
    - (re)compress all man pages with gzip -f -n -9
    - change all symlinks to point to .gz (if they don't already)
    - make all man pages be mode 644
    """
    requires = (
        ('ReadableDocs', policy.CONDITIONAL_SUBSEQUENT),
    )
    def _uncompress(self, dirname, names):
        for name in names:
            path = dirname + os.sep + name
            if name.endswith('.gz') and util.isregular(path):
                util.execute('gunzip ' + dirname + os.sep + name)
            if name.endswith('.bz2') and util.isregular(path):
                util.execute('bunzip2 ' + dirname + os.sep + name)

    def _touchup(self, dirname, names):
        """
        remove destdir, fix up modes, ensure that it is legal UTF-8
        """
        mode = os.lstat(dirname)[stat.ST_MODE]<commit_msg>use correct name in for NormalizeManPages in documentation<commit_after>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import os
import re
import stat
import tempfile

from conary.lib import util
from conary.build import policy


class NormalizeCompression(policy.DestdirPolicy):
    """
    NAME
    ====

    B{C{r.NormalizeCompression()}} - Compress files with maximum compression

    SYNOPSIS
    ========

    C{r.NormalizeCompression([I{filterexp}] I{exceptions=filterexp}])}

    DESCRIPTION
    ===========

    The C{r.NormalizeCompression()} policy compresses files with maximum
    compression, and without data which may change from invocation, to
    invocation.

    Recompresses .gz files with -9 -n, and .bz2 files with -9, to get maximum
    compression and avoid meaningless changes overpopulating the database.
    Ignores man/info pages, as they are encountered separately while making other
    changes to man/info pages later.

    EXAMPLES
    ========

    C{r.NormalizeCompression(exceptions='%(thistestdir)s/.*')}

    This package has test files that are tested byte-for-byte and
    cannot be modified at all and still pass the tests.
    """
    invariantexceptions = [
        '%(mandir)s/man.*/',
        '%(infodir)s/',
    ]
    invariantinclusions = [
        ('.*\.(gz|bz2)', None, stat.S_IFDIR),
    ]
    def doFile(self, path):
        m = self.recipe.magic[path]
        if not m:
            return

        # Note: uses external gzip/bunzip if they exist because a
        # pipeline is faster in a multiprocessing environment

        def _mktmp(fullpath):
            fd, path = tempfile.mkstemp('.temp', '', os.path.dirname(fullpath))
            os.close(fd)
            return path

        def _move(tmppath, fullpath):
            os.chmod(tmppath, os.lstat(fullpath).st_mode)
            os.rename(tmppath, fullpath)

        fullpath = self.macros.destdir+path
        if m.name == 'gzip' and \
           (m.contents['compression'] != '9' or 'name' in m.contents):
            tmppath = _mktmp(fullpath)
            util.execute('/bin/gzip -dc %s | /bin/gzip -f -n -9 > %s'
                         %(fullpath, tmppath))
            _move(tmppath, fullpath)
            del self.recipe.magic[path]
        if m.name == 'bzip' and m.contents['compression'] != '9':
            tmppath = _mktmp(fullpath)
            util.execute('/usr/bin/bzip2 -dc %s | /usr/bin/bzip2 -9 > %s'
                         %(fullpath, tmppath))
            _move(tmppath, fullpath)
            del self.recipe.magic[path]


class NormalizeManPages(policy.DestdirPolicy):
    """
    NAME
    ====

    B{C{r.NormalizeManPages()}} - Make all man pages follow sane system policy

    SYNOPSIS
    ========

    C{r.NormalizeManPages([I{filterexp}] I{exceptions=filterexp}])}

    DESCRIPTION
    ===========

    The C{r.NormalizeManPages()} policy makes all system manual pages
    follow sane system policy

    Note: This policy class is not called directly from recipes, and does not
    honor exceptions.

    Some of the following tasks are performed against system manual pages via
    C{r.NormalizeManPages}:

    - Fix all man pages' contents:
    - remove instances of C{/?%(destdir)s} from all man pages
    - C{.so foo.n} becomes a symlink to foo.n
    - (re)compress all man pages with gzip -f -n -9
    - change all symlinks to point to .gz (if they don't already)
    - make all man pages be mode 644
    """
    requires = (
        ('ReadableDocs', policy.CONDITIONAL_SUBSEQUENT),
    )
    def _uncompress(self, dirname, names):
        for name in names:
            path = dirname + os.sep + name
            if name.endswith('.gz') and util.isregular(path):
                util.execute('gunzip ' + dirname + os.sep + name)
            if name.endswith('.bz2') and util.isregular(path):
                util.execute('bunzip2 ' + dirname + os.sep + name)

    def _touchup(self, dirname, names):
        """
        remove destdir, fix up modes, ensure that it is legal UTF-8
        """
        mode = os.lstat(dirname)[stat.ST_MODE]
{'fraction_non_alphanumeric': 0.1100358812656301, 'fraction_numerical': 0.0058714798303794715, 'mean_word_length': 2.484090909090909, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4366626', 'n_tokens_mistral': 2852, 'n_tokens_neox': 2729, 'n_words': 890}