<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 Citrix Systems, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation; version 2.1 only. with the special
# exception on linking described in file LICENSE.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.

"""supplementalpack - create XCP supplemental packs"""

import warnings
warnings.simplefilter("ignore", DeprecationWarning)

from optparse import OptionParser

import md5
import os
import os.path
import re
import shutil
import stat
import subprocess
import sys
import tarfile
import tempfile
import xml.dom.minidom

def md5sum_file(fname):
        digest = md5.new()
        fh = open(fname)
        while (True):
            blk = fh.read(8192)
            if len(blk) == 0:
                break
            digest.update(blk)
        fh.close()
        return digest.hexdigest()

class Package:
    # allow creation of bzipped tar packages
    permit_legacy = False

    def __init__(self, fname):
        self.fname = fname
        try:
            self.size = str(os.stat(fname).st_size)
        except:
            raise SystemExit, "Cannot open " + fname
        self.md5 = md5sum_file(fname)

        filetype = subprocess.Popen(['file', '-k', '-b', fname],
                                    stdout=subprocess.PIPE).communicate()[0]
        if re.search('RPM', filetype):
            rpmname = subprocess.Popen(['rpm', '--nosignature', '-q', '--qf', '%{NAME}',
                                        '-p', fname], stdout=subprocess.PIPE).communicate()[0]
            rpmgroup = subprocess.Popen(['rpm', '--nosignature', '-q', '--qf', '%{GROUP}',
                                         '-p', fname], stdout=subprocess.PIPE).communicate()[0]
	    self.subtype = None
            if rpmname in ('kernel', 'kernel-xen', 'kernel-kdump'):
                self.subtype = 'kernel'
            if self.subtype != 'kernel' and rpmgroup.endswith('/Kernel'):
                self.type = 'driver-rpm'
                m = re.search('(.*)-modules-([^-]*)-(.*)', rpmname)
                if m:
                    self.label = m.group(1)
                    self.kernel = m.group(3) + m.group(2)
                else:
                    self.label = rpmname
                    self.kernel = 'any'
            else:
                self.type = 'rpm'
                self.label = rpmname
        elif re.search('bzip2', filetype):
            self.type = 'tbz2'
            self.label = os.path.basename(fname)
	    self.root = '/'
        else:
            self.type = 'unknown'

    # validate a package
    def check(self):
        if self.type == 'unknown' or (self.type == 'tbz2' and not self.permit_legacy):
            raise SystemExit, "Error: unsupported package type " + self.fname

        if self.type == 'rpm':
            p = subprocess.Popen(['rpm', '--nosignature', '-qlp', self.fname],
                                 stdout=subprocess.PIPE)
            while True:
                l = p.stdout.readline().strip()
                if len(l) == 0:
                    break
                if (l.startswith('/lib/modules/') or l.startswith('/boot/vmlinu')) and self.subtype != 'kernel':
                    raise SystemExit, "Error: unsupported file %s in %s" % (l, self.fname)
            p.wait()
        elif self.type == 'driver-rpm':
            if self.flavour() not in ('any', 'kdump', 'xen'):
                raise SystemExit, "Error: unexpected kernel suffix (%s)" % self.kernel
            p = subprocess.Popen(['rpm', '--nosignature', '-qlp', self.fname],
                                 stdout=subprocess.PIPE)
            while True:
                l = p.stdout.readline().strip()
                if len(l) == 0:
                    break
                if self.kernel == 'any':
                    # firmware / udev
                    paths = ('/etc/udev/rules.d', '/lib/firmware', '/etc',
                             '/usr/share/doc')
                    if not True in map(lambda x: l == x or l.startswith(x+'/'), paths):
                        raise SystemExit, "Error: unsupported file %s in %s" % (l, self.fname)
                else:
                    # kernel modules
                    if not l.startswith('/lib/modules/'+self.kernel+'/extra/'):
                        raise SystemExit, "Error: unsupported file %s in %s" % (l, self.fname)
            p.wait()

    def toxml(self, doc):
        common_attrs = ('label', 'type', 'size', 'md5')

        pe = doc.createElement("package")
        for a in common_attrs:
            pe.setAttribute(a, self.__dict__[a])
        if self.type == 'driver-rpm':
            pe.setAttribute('kernel', self.kernel)
	elif self.type == 'tbz2':
	    pe.setAttribute('root', self.root)
        pe.appendChild(doc.createTextNode(os.path.basename(self.fname)))

        return pe

    def flavour(self):
        if self.type == 'driver-rpm':
            m = re.search('([a-z]+)$', self.kernel)
            return m.group(1)
        else:
            return ''
<commit_msg>Handle legacy driver packages that place files down /lib/modules/.../kernel/<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 Citrix Systems, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation; version 2.1 only. with the special
# exception on linking described in file LICENSE.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.

"""supplementalpack - create XCP supplemental packs"""

import warnings
warnings.simplefilter("ignore", DeprecationWarning)

from optparse import OptionParser

import md5
import os
import os.path
import re
import shutil
import stat
import subprocess
import sys
import tarfile
import tempfile
import xml.dom.minidom

def md5sum_file(fname):
        digest = md5.new()
        fh = open(fname)
        while (True):
            blk = fh.read(8192)
            if len(blk) == 0:
                break
            digest.update(blk)
        fh.close()
        return digest.hexdigest()

class Package:
    # allow creation of bzipped tar packages
    permit_legacy = False

    driver_whitelist = ('md3000-rdac')

    def __init__(self, fname):
        self.fname = fname
        try:
            self.size = str(os.stat(fname).st_size)
        except:
            raise SystemExit, "Cannot open " + fname
        self.md5 = md5sum_file(fname)

        filetype = subprocess.Popen(['file', '-k', '-b', fname],
                                    stdout=subprocess.PIPE).communicate()[0]
        if re.search('RPM', filetype):
            rpmname = subprocess.Popen(['rpm', '--nosignature', '-q', '--qf', '%{NAME}',
                                        '-p', fname], stdout=subprocess.PIPE).communicate()[0]
            rpmgroup = subprocess.Popen(['rpm', '--nosignature', '-q', '--qf', '%{GROUP}',
                                         '-p', fname], stdout=subprocess.PIPE).communicate()[0]
	    self.subtype = None
            if rpmname in ('kernel', 'kernel-xen', 'kernel-kdump'):
                self.subtype = 'kernel'
            if self.subtype != 'kernel' and rpmgroup.endswith('/Kernel'):
                self.type = 'driver-rpm'
                m = re.search('(.*)-modules-([^-]*)-(.*)', rpmname)
                if m:
                    self.label = m.group(1)
                    self.kernel = m.group(3) + m.group(2)
                else:
                    self.label = rpmname
                    self.kernel = 'any'
            else:
                self.type = 'rpm'
                self.label = rpmname
        elif re.search('bzip2', filetype):
            self.type = 'tbz2'
            self.label = os.path.basename(fname)
	    self.root = '/'
        else:
            self.type = 'unknown'

    # validate a package
    def check(self):
        if self.type == 'unknown' or (self.type == 'tbz2' and not self.permit_legacy):
            raise SystemExit, "Error: unsupported package type " + self.fname

        if self.type == 'rpm':
            p = subprocess.Popen(['rpm', '--nosignature', '-qlp', self.fname],
                                 stdout=subprocess.PIPE)
            while True:
                l = p.stdout.readline().strip()
                if len(l) == 0:
                    break
                if (l.startswith('/lib/modules/') or l.startswith('/boot/vmlinu')) and self.subtype != 'kernel':
                    raise SystemExit, "Error: unsupported file %s in %s" % (l, self.fname)
            p.wait()
        elif self.type == 'driver-rpm':
            if self.flavour() not in ('any', 'kdump', 'xen'):
                raise SystemExit, "Error: unexpected kernel suffix (%s)" % self.kernel
            p = subprocess.Popen(['rpm', '--nosignature', '-qlp', self.fname],
                                 stdout=subprocess.PIPE)
            while True:
                l = p.stdout.readline().strip()
                if len(l) == 0:
                    break
                if self.kernel == 'any':
                    # firmware / udev
                    paths = ('/etc/udev/rules.d', '/lib/firmware', '/etc',
                             '/usr/share/doc')
                    if not True in map(lambda x: l == x or l.startswith(x+'/'), paths):
                        raise SystemExit, "Error: unsupported file %s in %s" % (l, self.fname)
                else:
                    # kernel modules
                    if not l.startswith('/lib/modules/'+self.kernel+'/extra/'):
                        if l.startswith('/lib/modules/'+self.kernel+'/kernel/') and \
                                        self.label not in self.driver_whitelist:
                            raise SystemExit, "Error: unsupported file %s in %s" % (l, self.fname)
            p.wait()

    def toxml(self, doc):
        common_attrs = ('label', 'type', 'size', 'md5')

        pe = doc.createElement("package")
        for a in common_attrs:
            pe.setAttribute(a, self.__dict__[a])
        if self.type == 'driver-rpm':
            pe.setAttribute('kernel', self.kernel)
	elif self.type == 'tbz2':
	    pe.setAttribute('root', self.root)
        pe.appendChild(doc.createTextNode(os.path.basename(self.fname)))

        return pe

    def flavour(self):
        if self.type == 'driver-rpm':
            m = re.search('([a-z]+)$', self.kernel)
            return m.group(1)
        else:
            return ''

{'fraction_non_alphanumeric': 0.13001093294460642, 'fraction_numerical': 0.0058309037900874635, 'mean_word_length': 1.6997048696507624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '767053', 'n_tokens_mistral': 3249, 'n_tokens_neox': 3070, 'n_words': 859}