<commit_before>#!/usr/bin/env python

import sys
import os.path
from subprocess import call
import urlparse
import subprocess
import re
import glob
import shutil

SOURCESDIR = "./SOURCES"
SRPMSDIR = "./SRPMS"
SPECSDIR = "./SPECS"

number_skipped = 0
number_fetched = 0

# HACK: Monkey-patch urlparse to understand git:// URLs
# This is not needed for more modern Pythons
#    http://bugs.python.org/issue7904
urlparse.uses_netloc.append('git')

def rewrite_to_distfiles(url):
    """
    Rewrites url to refer to the local distfiles cache.
    """
    basename = url.split("/")[-1]
    return "file:///distfiles/ocaml/" + basename
    

def fetch_url(url, rewrite=None):
    """ 
    Fetches a url, rewriting it using 'rewrite' if it exists

    Only fetches if the target file doesn't already exist.

    Returns 1 if it actually fetched something, otherwise 0.
    """
    if rewrite:
        url = rewrite(url)
    basename = url.split("/")[-1]
    final_path = os.path.join(SOURCESDIR, basename)
    if os.path.exists(final_path):
        return 0
    else:
        if call(["curl", "-k", "-L", "-o", final_path, url]) != 0:
            print "Error downloading '%s'" % url
            sys.exit(1)
        return 1

def latest_git_tag(path):
    """
    Returns numeric version tag closest to HEAD in the repository.
    """
    # We expect path to be a full git url pointing at a path on the local host
    # We only need the path
    (scheme, host, path, _, _, _) = urlparse.urlparse(path)
    assert scheme == "git"
    assert host == ""

    description = subprocess.Popen(
        ["git", "--git-dir=%s/.git" % path,
         "describe", "--tags"],
        stdout=subprocess.PIPE).communicate()[0].strip()
    m = re.search("[^0-9]*", description)
    l = len(m.group())
    return description[l:].replace('-', '+')


def fetch_git_source(path):
    """
    Fetches an archive of a git repository.

    Returns a version string, the tarball's prefix, and the pathname of
    the tarball.
    """

    # We expect path to be a full git url pointing at a path on the local host
    # We only need the path
    (scheme, host, path, _, _, _) = urlparse.urlparse(path)
    assert scheme == "git"
    assert host == ""

    version = latest_git_tag(path)
    basename = path.split("/")[-1]
    [os.remove(f)
        for f in os.listdir('SOURCES')
        if re.search('^(%s\.tar)(\.gz)?$' % basename, f)]
    call(["git", "--git-dir=%s/.git" % path, "archive",
          "--prefix=%s-%s/" % (basename, version), "HEAD", "-o",
          "%s/%s-%s.tar.gz" % (SOURCESDIR, basename, version)])
    return (version, "%s-%s.tar.gz" % (basename, version))


def sources_from_spec(spec_path):
    """
    Extracts all source URLS from the spec file at spec_path.

    Returns a list of source URLs with RPM macros expanded.
    """
    sources = []
    lines = subprocess.Popen(
        ["./spectool", "--list-files", "--sources", spec_path],
         stdout=subprocess.PIPE).communicate()[0].strip().split("\n")
    for l in lines:
        m = re.match("^([Ss]ource\d*):\s+(\S+)$", l)
	assert m
        sources.append(m.groups()[1])
    return sources 


def preprocess_spec(spec_path, version, sources):
    """
    Preprocesses a spec file containing placeholders and
    returns the path to the resulting file.
    """

    # Rewrite the @VERSION@ placeholder in the spec and 
    # add .tar.gz to the source URL, so rpmbuild finds the file.
    spec_contents = subprocess.Popen(
        ["sed", "-e", "s/@VERSION@/%s/g" % version, 
         "-e", "s$%s$%s-%%{version}.tar.gz$g" % (sources[0], sources[0]),
         "%s" % spec_path],
        stdout=subprocess.PIPE).communicate()[0]
    f = open(os.path.splitext(spec_path)[0], "w")
    f.write(spec_contents)
    f.close()


def prepare_srpm(spec_path):
    """
    Downloads sources needed to build an SRPM from the spec file
    at spec_path.   Pre-processes the spec file, if needed.
    """
    global number_skipped, number_fetched

    # check the .spec file exists, or .spec.in if we're processing the spec
    if not(os.path.exists(spec_path)):
        print "%s doesn't exist" % spec_path
        sys.exit(1)

    # Pull out the source list.   If the spec file pulls its sources 
    # from a Git repository, we need to prepreprocess the spec file 
    # to fill in the latest version tag from the repository.
    sources = sources_from_spec(spec_path)
    assert sources

    if spec_path.endswith('.in'):
        print "Configuring package with spec file: %s" % spec_path
        version, filename = fetch_git_source(sources[0])
        preprocess_spec(spec_path, version, sources)
        return

    for source in sources:
        (scheme, _, path, _, _, _) = urlparse.urlparse(
            source)

        if scheme in ['file', 'http', 'https']:
            result = fetch_url(source, rewrite_to_distfiles)
            number_fetched += result
            number_skipped += (1 - result)


def build_srpm(spec_path):
    """
    Builds an SRPM from the spec file at spec_path.

    Assumes that all source files have already been downloaded to 
    the rpmbuild sources directory, and are correctly named. <commit_msg>configure: Separate repository version query from archive download<commit_after>#!/usr/bin/env python

import sys
import os.path
from subprocess import call
import urlparse
import subprocess
import re
import glob
import shutil

SOURCESDIR = "./SOURCES"
SRPMSDIR = "./SRPMS"
SPECSDIR = "./SPECS"

number_skipped = 0
number_fetched = 0

# HACK: Monkey-patch urlparse to understand git:// URLs
# This is not needed for more modern Pythons
#    http://bugs.python.org/issue7904
urlparse.uses_netloc.append('git')

def rewrite_to_distfiles(url):
    """
    Rewrites url to refer to the local distfiles cache.
    """
    basename = url.split("/")[-1]
    return "file:///distfiles/ocaml/" + basename
    

def fetch_url(url, rewrite=None):
    """ 
    Fetches a url, rewriting it using 'rewrite' if it exists

    Only fetches if the target file doesn't already exist.

    Returns 1 if it actually fetched something, otherwise 0.
    """
    if rewrite:
        url = rewrite(url)
    basename = url.split("/")[-1]
    final_path = os.path.join(SOURCESDIR, basename)
    if os.path.exists(final_path):
        return 0
    else:
        if call(["curl", "-k", "-L", "-o", final_path, url]) != 0:
            print "Error downloading '%s'" % url
            sys.exit(1)
        return 1

def latest_git_tag(path):
    """
    Returns numeric version tag closest to HEAD in the repository.
    """
    # We expect path to be a full git url pointing at a path on the local host
    # We only need the path
    (scheme, host, path, _, _, _) = urlparse.urlparse(path)
    assert scheme == "git"
    assert host == ""

    description = subprocess.Popen(
        ["git", "--git-dir=%s/.git" % path,
         "describe", "--tags"],
        stdout=subprocess.PIPE).communicate()[0].strip()
    m = re.search("[^0-9]*", description)
    l = len(m.group())
    return description[l:].replace('-', '+')


def fetch_git_source(path, version, archive_name):
    """
    Fetches an archive of HEAD of the git repository at path.
    Produces a tarball called 'archive_name' in SOURCESDIR,
    which when unpacked will produce a directory called
    "reponame-version".   This is similar to GitHub's archive
    URLs.
    """

    # We expect path to be a full git url pointing at a path on the local host
    # We only need the path
    (scheme, host, path, _, _, _) = urlparse.urlparse(path)
    assert scheme == "git"
    assert host == ""

    basename = path.split("/")[-1]
    [os.remove(f)
        for f in os.listdir('SOURCES')
        if re.search('^(%s\.tar)(\.gz)?$' % basename, f)]
    call(["git", "--git-dir=%s/.git" % path, "archive",
          "--prefix=%s-%s/" % (basename, version), "HEAD", "-o",
          "%s/%s" % (SOURCESDIR, archive_name)])


def name_from_spec(spec_path):
    """
    Returns the base name of the packages defined in the spec file at spec_path.
    """
    f = open(spec_path)
    lines = f.readlines()
    f.close()

    name = [l.strip() for l in lines 
            if l.strip().lower().startswith('name:')][0].split(':')[1].strip()
    return name 


def sources_from_spec(spec_path):
    """
    Extracts all source URLS from the spec file at spec_path.

    Returns a list of source URLs with RPM macros expanded.
    """
    sources = []
    lines = subprocess.Popen(
        ["./spectool", "--list-files", "--sources", spec_path],
         stdout=subprocess.PIPE).communicate()[0].strip().split("\n")
    for l in lines:
        m = re.match("^([Ss]ource\d*):\s+(\S+)$", l)
	assert m
        sources.append(m.groups()[1])
    return sources 


def preprocess_spec(spec_path, version, tarball_name):
    """
    Preprocesses a spec file containing placeholders and
    returns the path to the resulting file.
    """

    # Rewrite the @VERSION@ placeholder in the spec and 
    # add .tar.gz to the source URL, so rpmbuild finds the file.
    spec_contents = subprocess.Popen(
        ["sed", "-e", "s/@VERSION@/%s/g" % version, 
         "-e", "s/Source0:.*/Source0: %s/g" % tarball_name,
         "%s" % spec_path],
        stdout=subprocess.PIPE).communicate()[0]
    f = open(os.path.splitext(spec_path)[0], "w")
    f.write(spec_contents)
    f.close()


def prepare_srpm(spec_path):
    """
    Downloads sources needed to build an SRPM from the spec file
    at spec_path.   Pre-processes the spec file, if needed.
    """
    global number_skipped, number_fetched

    # check the .spec file exists, or .spec.in if we're processing the spec
    if not(os.path.exists(spec_path)):
        print "%s doesn't exist" % spec_path
        sys.exit(1)

    # Pull out the source list.   If the spec file pulls its sources 
    # from a Git repository, we need to prepreprocess the spec file 
    # to fill in the latest version tag from the repository.
    sources = sources_from_spec(spec_path)
    assert sources

    if spec_path.endswith('.in'):
        print "Configuring package with spec file: %s" % spec_path
        version = latest_git_tag(sources[0])
        tarball_name = "%s-%s.tar.gz" % (name_from_spec(spec_path), version)
        fetch_git_source(sources[0], version, tarball_name)
        preprocess_spec(spec_path, version, tarball_name)
        return

    for source in sources:
        (scheme, _, path, _, _, _) = urlparse.urlparse(
            source)

        if scheme in ['file', 'http', 'https']:
            result = fetch_url(source, rewrite_to_distfiles)
            number_fetched += result
            number_skipped += (1 - result)


def build_srpm(spec_path):
    """
    Builds an SRPM from the spec file at spec_path.

    Assumes that all source files have already been downloaded to 
    the rpmbuild sources directory, and are correctly named. 
{'fraction_non_alphanumeric': 0.1309294430681714, 'fraction_numerical': 0.005229837599779796, 'mean_word_length': 2.701188455008489, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '560845', 'n_tokens_mistral': 3575, 'n_tokens_neox': 3409, 'n_words': 1076}