<commit_before>#!/usr/bin/env python

"""
Builds SRPMs for the tarballs or Git repositories in <component-specs-dir>.
"""

import argparse
import sys
import os.path
import urlparse
import re
import glob
import shutil
from planex.globals import (BUILD_ROOT_DIR, SPECS_DIR, SOURCES_DIR, SRPMS_DIR,
                            SPECS_GLOB, REPOS_PATH, HASHFN)
import planex.spec
from planex.util import (bcolors, run, dump_cmds)
from planex import sources

GITHUB_MIRROR = "~/github_mirror"

manifest = {}

def name_from_spec(spec_path):
    """
    Returns the base name of the packages defined in the spec file at spec_path.
    Ideally we would do this using the Python RPM library, but the version in
    CentOS 5 doesn't expose it.
    """
    spec = open(spec_path)
    lines = spec.readlines()
    spec.close()

    name = [l.strip() for l in lines
            if l.strip().lower().startswith('name:')][0].split(':')[1].strip()
    return name


def check_spec_name(spec_path):
    """
    The spec file name should match the base name of the packages it produces.
    Exit with an error if this is not the case.
    """
    pkg_name = name_from_spec(spec_path)
    if re.sub(r".spec(.in)?$", "", os.path.basename(spec_path)) != pkg_name:
        sys.stderr.write( "error: spec file name '%s' "
                          "does not match package name '%s'\n" %
                          (spec_path, pkg_name))
        sys.exit(1)


def sources_from_spec(spec_path):
    """
    Extracts all source URLS from the spec file at spec_path.

    Returns a list of source URLs with RPM macros expanded.
    """
    spec = planex.spec.Spec(spec_path)
    return spec.source_urls()


def preprocess_spec(spec_in_path, spec_out_path, scmsources, source_mapping):
    """
    Preprocesses a spec file containing placeholders.
    Writes the result to the same filename, with the '.in' extension
    stripped, in spec_out_path.
    """
    assert spec_in_path.endswith('.in')

#    print "preprocess_spec: mapping=%s" % source_mapping
    spec_in = open(spec_in_path)
    spec_contents = spec_in.readlines()
    spec_in.close()

    output_filename = os.path.basename(spec_in_path)[:-len(".in")]
    spec_out = open(os.path.join(spec_out_path, output_filename), "w")

    spec_basename = spec_in_path.split("/")[-1]

    subs = {}

    for (index,source) in enumerate(scmsources):
        subs['source%d_version' % index] = source.version
        subs['source%d_hash' % index] = source.scmhash

    subs.update({
	"version" : "+".join([source.version for source in scmsources]),
        "release" : "1%{?extrarelease}" })

    for line in spec_contents:
        match = re.match(r'^([Ss]ource\d*:\s+)(.+)\n', line)
# TODO: strip whitespace from match.group(2)
        if match and match.group(2) in source_mapping:
#            print "Got a source mapping"
            line = match.group(1) + source_mapping[match.group(2)] + "\n"
            print "  " + spec_basename + ": mapping %s to %s" % (match.group(2), 
                                        source_mapping[match.group(2)])

        match = re.match(r'^(%define planex_)([^\s]+)(.+)\n', line)
        if match:
            line = match.group(1) + match.group(2) + " " + subs[match.group(2)] + "\n"

        spec_out.write(line)

    spec_out.close()


def prepare_srpm(spec_path):
    """
    Downloads sources needed to build an SRPM from the spec file
    at spec_path.
    """
    # check the .spec file exists, or .spec.in if we're processing the spec
    if not(os.path.exists(spec_path)):
        print "%s doesn't exist" % spec_path
        sys.exit(1)

    # Pull out the source list.   If the spec file pulls its sources
    # from a Git repository, we need to prepreprocess the spec file
    # to fill in the latest version tag from the repository.
    allsources = sources_from_spec(spec_path)
    if allsources == []:
        print "Failed to get sources for %s" % spec_path
        sys.exit(1)

    for source in allsources:
        sources.Source(source).archive()

def get_hashes(ty):
    spec_files = glob.glob(os.path.join(SPECS_DIR,"*"))
    sources_files = glob.glob(os.path.join(SOURCES_DIR,"*"))
    all_files = spec_files + sources_files
    if ty=="md5":
        cmd = ["md5sum"] + all_files
    elif ty=="sha256":
        cmd = ["sha256sum"] + all_files
    else:
        print "Invalid hash type"
        raise Exception
    result = run(cmd)['stdout'].strip().split('\n')
    def fix(x):
        words = x.split()
        if len(words) == 2:
            fname = words[1].split("/")[-1]
            return (fname,words[0])
        else:
            return None
    fixed = [fix(x) for x in result]
    return dict(fixed)       

def ensure_existing_ok(hashes, spec_path):
    pkg_name = name_from_spec(spec_path)

    one_correct = False

    for srpm in glob.glob(os.path.join(SRPMS_DIR, '%s-*.src.rpm' % pkg_name)):
        # Check it's for the right package:
        cmd = ["rpm","-qp",srpm, "--qf", "%{name}"]
        result = run(cmd)['stdout'].strip().split('\n')
        
        if result[0] == pkg_name:
            cmd = ["rpm","--dump","-qp",srpm]
            result = run(cmd)['stdout'].strip().split('\n')
            ok = True
            for line in result:
                split = line.split()
                fname = split[0]
                thishash=split[3]
                if hashes[fname] != thishash:
                    ok = False

            if not ok:
                print bcolors.WARNING + ("WARNING: Removing SRPM '%s' (hash mismatch with desired)" % srpm) + bcolors.ENDC
                os.remove(srpm)
            else:
                one_correct = True

    return one_correct

def build_srpm(hashes, spec_path):
    """
    Builds an SRPM from the spec file at spec_path.

    Assumes that all source files have already been downloaded to
    the rpmbuild sources directory, and are correctly named.
    """
    is_ok = ensure_existing_ok(hashes, spec_path)<commit_msg>configure: Handle missing source files gracefully<commit_after>#!/usr/bin/env python

"""
Builds SRPMs for the tarballs or Git repositories in <component-specs-dir>.
"""

import argparse
import sys
import os.path
import urlparse
import re
import glob
import shutil
from planex.globals import (BUILD_ROOT_DIR, SPECS_DIR, SOURCES_DIR, SRPMS_DIR,
                            SPECS_GLOB, REPOS_PATH, HASHFN)
import planex.spec
from planex.util import (bcolors, run, dump_cmds)
from planex import sources

GITHUB_MIRROR = "~/github_mirror"

manifest = {}

def name_from_spec(spec_path):
    """
    Returns the base name of the packages defined in the spec file at spec_path.
    Ideally we would do this using the Python RPM library, but the version in
    CentOS 5 doesn't expose it.
    """
    spec = open(spec_path)
    lines = spec.readlines()
    spec.close()

    name = [l.strip() for l in lines
            if l.strip().lower().startswith('name:')][0].split(':')[1].strip()
    return name


def check_spec_name(spec_path):
    """
    The spec file name should match the base name of the packages it produces.
    Exit with an error if this is not the case.
    """
    pkg_name = name_from_spec(spec_path)
    if re.sub(r".spec(.in)?$", "", os.path.basename(spec_path)) != pkg_name:
        sys.stderr.write( "error: spec file name '%s' "
                          "does not match package name '%s'\n" %
                          (spec_path, pkg_name))
        sys.exit(1)


def sources_from_spec(spec_path):
    """
    Extracts all source URLS from the spec file at spec_path.

    Returns a list of source URLs with RPM macros expanded.
    """
    spec = planex.spec.Spec(spec_path)
    return spec.source_urls()


def preprocess_spec(spec_in_path, spec_out_path, scmsources, source_mapping):
    """
    Preprocesses a spec file containing placeholders.
    Writes the result to the same filename, with the '.in' extension
    stripped, in spec_out_path.
    """
    assert spec_in_path.endswith('.in')

#    print "preprocess_spec: mapping=%s" % source_mapping
    spec_in = open(spec_in_path)
    spec_contents = spec_in.readlines()
    spec_in.close()

    output_filename = os.path.basename(spec_in_path)[:-len(".in")]
    spec_out = open(os.path.join(spec_out_path, output_filename), "w")

    spec_basename = spec_in_path.split("/")[-1]

    subs = {}

    for (index,source) in enumerate(scmsources):
        subs['source%d_version' % index] = source.version
        subs['source%d_hash' % index] = source.scmhash

    subs.update({
	"version" : "+".join([source.version for source in scmsources]),
        "release" : "1%{?extrarelease}" })

    for line in spec_contents:
        match = re.match(r'^([Ss]ource\d*:\s+)(.+)\n', line)
# TODO: strip whitespace from match.group(2)
        if match and match.group(2) in source_mapping:
#            print "Got a source mapping"
            line = match.group(1) + source_mapping[match.group(2)] + "\n"
            print "  " + spec_basename + ": mapping %s to %s" % (match.group(2), 
                                        source_mapping[match.group(2)])

        match = re.match(r'^(%define planex_)([^\s]+)(.+)\n', line)
        if match:
            line = match.group(1) + match.group(2) + " " + subs[match.group(2)] + "\n"

        spec_out.write(line)

    spec_out.close()


def prepare_srpm(spec_path):
    """
    Downloads sources needed to build an SRPM from the spec file
    at spec_path.
    """
    # check the .spec file exists, or .spec.in if we're processing the spec
    if not(os.path.exists(spec_path)):
        print "%s doesn't exist" % spec_path
        sys.exit(1)

    # Pull out the source list.   If the spec file pulls its sources
    # from a Git repository, we need to prepreprocess the spec file
    # to fill in the latest version tag from the repository.
    allsources = sources_from_spec(spec_path)
    if allsources == []:
        print "Failed to get sources for %s" % spec_path
        sys.exit(1)

    for source in allsources:
        sources.Source(source).archive()

def get_hashes(ty):
    spec_files = glob.glob(os.path.join(SPECS_DIR,"*"))
    sources_files = glob.glob(os.path.join(SOURCES_DIR,"*"))
    all_files = spec_files + sources_files
    if ty=="md5":
        cmd = ["md5sum"] + all_files
    elif ty=="sha256":
        cmd = ["sha256sum"] + all_files
    else:
        print "Invalid hash type"
        raise Exception
    result = run(cmd)['stdout'].strip().split('\n')
    def fix(x):
        words = x.split()
        if len(words) == 2:
            fname = words[1].split("/")[-1]
            return (fname,words[0])
        else:
            return None
    fixed = [fix(x) for x in result]
    return dict(fixed)       

def ensure_existing_ok(hashes, spec_path):
    pkg_name = name_from_spec(spec_path)

    one_correct = False

    for srpm in glob.glob(os.path.join(SRPMS_DIR, '%s-*.src.rpm' % pkg_name)):
        # Check it's for the right package:
        cmd = ["rpm","-qp",srpm, "--qf", "%{name}"]
        result = run(cmd)['stdout'].strip().split('\n')
        
        if result[0] == pkg_name:
            cmd = ["rpm","--dump","-qp",srpm]
            result = run(cmd)['stdout'].strip().split('\n')
            ok = True
            for line in result:
                split = line.split()
                fname = split[0]
                thishash=split[3]
                if fname not in hashes or hashes[fname] != thishash:
                    ok = False

            if not ok:
                print bcolors.WARNING + ("WARNING: Removing SRPM '%s' (hash mismatch with desired)" % srpm) + bcolors.ENDC
                os.remove(srpm)
            else:
                one_correct = True

    return one_correct

def build_srpm(hashes, spec_path):
    """
    Builds an SRPM from the spec file at spec_path.

    Assumes that all source files have already been downloaded to
    the rpmbuild sources directory, and are correctly named.
    """
    is_ok = ensure_existing_ok(hashes, spec_path)
{'fraction_non_alphanumeric': 0.12455279141359514, 'fraction_numerical': 0.005324902238122972, 'mean_word_length': 2.47298468650679, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '191038', 'n_tokens_mistral': 3933, 'n_tokens_neox': 3795, 'n_words': 1057}