<commit_before>#
# Copyright (c) 2009 rPath, Inc.
#
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

"""
Module for interacting with a directory of packages.
"""

import os
import logging

from conary import rpmhelper

log = logging.getLogger('updatebot.pkgsource')

from repomd.packagexml import _Package
from rpmutils import header as rpmheader

from updatebot.lib import urlwalker
from updatebot.pkgsource.yumsource import YumSource as PackageSource

class Package(_Package):
    """
    Class for represnting package data.
    """

    def __init__(self, *args, **kwargs):
        _Package.__init__(self, *args)
        for key, val in kwargs.iteritems():
            setattr(self, key, val)

    def getNevra(self):
        """
        Return the name, epoch, version, release, and arch the package.
        """

        return (self.name, self.epoch, self.version, self.release, self.arch)

    def getConaryVersion(self):
        """
        Get the conary version of a source package.
        """

        assert self.arch == 'src'
        filename = os.path.basename(self.location)
        nameVerRelease = ".".join(filename.split(".")[:-2])
        ver = "_".join(nameVerRelease.split("-")[-2:])
        return ver


class Client(object):
    """
    Client class for walking package tree.
    """

    walkMethod = os.walk

    def __init__(self, path):
        self._path = path

    def getPackageDetail(self):
        """
        Walk the specified path to find rpms.
        """

        idx = 0
        for path, dirs, files in self.walkMethod(self._path):
            for f in files:
                if f.endswith('.rpm'):
                    idx += 1
                    if idx % 50 == 0:
                        log.info('indexing %s' % idx)
                    yield self._index(os.path.join(path, f))

    def _index(self, rpm):
        """
        Index an individual rpm.
        """

        fh = rpmheader.SeekableStream(rpm)
        h = rpmhelper.readHeader(fh)
        name = h[rpmhelper.NAME]
        epoch = h.get(rpmhelper.EPOCH, None)
        if isinstance(epoch, (list, tuple)):
            assert len(epoch) == 1
            epoch = str(epoch[0])
        version = h[rpmhelper.VERSION]
        release = h[rpmhelper.RELEASE]
        arch = h.isSource and 'src' or h[rpmhelper.ARCH]
        sourcename = h.get(rpmhelper.SOURCERPM, None)

        basename = os.path.basename(rpm)
        pkg = Package(name=name,
                      epoch=epoch,
                      version=version,
                      release=release,
                      arch=arch,
                      sourcerpm=sourcename,
                      location=basename)
        return pkg


class UrlClient(Client):
    """
    Url based client.
    """

    walkMethod = urlwalker.walk


class RpmSource(PackageSource):
    """
    Walk a directory, find rpms, index them.<commit_msg>switch to using a yum backed source<commit_after>#
# Copyright (c) 2009 rPath, Inc.
#
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

"""
Module for interacting with a directory of packages.
"""

import os
import logging

from conary import rpmhelper

log = logging.getLogger('updatebot.pkgsource')

from repomd.packagexml import _Package
from rpmutils import header as rpmheader

from updatebot.lib import urlwalker
from updatebot.pkgsource.yumsource import YumSource as PackageSource

class Package(_Package):
    """
    Class for represnting package data.
    """

    def __init__(self, *args, **kwargs):
        _Package.__init__(self, *args)
        for key, val in kwargs.iteritems():
            setattr(self, key, val)

    def getNevra(self):
        """
        Return the name, epoch, version, release, and arch the package.
        """

        return (self.name, self.epoch, self.version, self.release, self.arch)

    def getConaryVersion(self):
        """
        Get the conary version of a source package.
        """

        assert self.arch == 'src'
        filename = os.path.basename(self.location)
        nameVerRelease = ".".join(filename.split(".")[:-2])
        ver = "_".join(nameVerRelease.split("-")[-2:])
        return ver


class Client(object):
    """
    Client class for walking package tree.
    """

    walkMethod = os.walk

    def __init__(self, path):
        self._path = path

    def getPackageDetail(self):
        """
        Walk the specified path to find rpms.
        """

        idx = 0
        for path, dirs, files in self.walkMethod(self._path):
            for f in files:
                if f.endswith('.rpm'):
                    idx += 1
                    if idx % 50 == 0:
                        log.info('indexing %s' % idx)
                    yield self._index(os.path.join(path, f))

    def _index(self, rpm):
        """
        Index an individual rpm.
        """

        fh = rpmheader.SeekableStream(rpm)
        h = rpmhelper.readHeader(fh)
        name = h[rpmhelper.NAME]
        epoch = h.get(rpmhelper.EPOCH, None)
        if isinstance(epoch, (list, tuple)):
            assert len(epoch) == 1
            epoch = str(epoch[0])
        version = h[rpmhelper.VERSION]
        release = h[rpmhelper.RELEASE]
        arch = h.isSource and 'src' or h[rpmhelper.ARCH]
        sourcename = h.get(rpmhelper.SOURCERPM, None)

        basename = os.path.basename(rpm)
        pkg = Package(name=name,
                      epoch=epoch,
                      version=version,
                      release=release,
                      arch=arch,
                      sourcerpm=sourcename,
                      location=basename)
        return pkg


class UrlClient(Client):
    """
    Url based client.
    """

    walkMethod = urlwalker.walk


class RpmSource(PackageSource):
    PkgClass = Package

    def iterPackageSet(self):
        """
        Iterate over the set of source packages.
        """

        for srcPkg, binPkgs in self.srcPkgMap.iteritems():
            if not len(binPkgs):
                continue
            yield (srcPkg.name, srcPkg.getConaryVersion())

    def _excludeLocation(self, location):
        # FIXME: This should not be hard coded. There needs to be a config
        #        option for exclusions.
        path = os.path.dirname(location)
        if 'VT' in path or 'Cluster' in path:
            return True

class _RpmSource(PackageSource):
    """
    Walk a directory, find rpms, index them.
{'fraction_non_alphanumeric': 0.09933774834437085, 'fraction_numerical': 0.004595215569671577, 'mean_word_length': 2.082049146189088, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6938515', 'n_tokens_mistral': 2181, 'n_tokens_neox': 2069, 'n_words': 663}