<commit_before>import os
import sys
import multiprocessing
import subprocess
import signal
import urlparse
from yum.Errors import RepoError
from pakrat import util, log, repotools

__version__ = '0.0.7'

def sync(basedir, repos=[], repodirs=[], repofiles=[], repoversion=None, delete=False):
    util.validate_basedir(basedir)
    util.validate_repos(repos)
    util.validate_repofiles(repofiles)
    util.validate_repodirs(repodirs)
    util.validate_repos(repos)

    if repoversion:
        delete = False

    for file in repofiles:
        for filerepo in repotools.from_file(file):
            repos.append(filerepo)

    for dir in repodirs:
        for dirrepo in repotools.from_dir(dir):
            repos.append(dirrepo)

    processes = []
    for repo in repos:
        dest = util.get_repo_dir(basedir, repo.id)
        p = multiprocessing.Process(target=sync_repo, args=(repo, dest, repoversion, delete))
        p.start()
        processes.append(p)

    def cleanup(*args):
        for p in processes:
            os.kill(p.pid, signal.SIGKILL)
        sys.exit(1)

    signal.signal(signal.SIGINT, cleanup)
    signal.signal(signal.SIGTERM, cleanup)

    complete_count = 0
    while True:
        for p in processes:
            if not p.is_alive():
                complete_count += 1
        if complete_count == len(processes):
            break

def sync_repo(repo, dest, version, delete=False):
    try:
        packages_dir = util.get_packages_dir(dest)
        yb = util.get_yum()
        repo = repotools.set_path(repo, packages_dir)
        yb.repos.add(repo)
        yb.repos.enableRepo(repo.id)
        packages = []
        for package in yb.doPackageLists(pkgnarrow='available', showdups=False):
            packages.append(package)
    except RepoError, e:<commit_msg>Fixed issues where installed packages were ignored, and where only the newest packages would be returned if there were multiple versions in the repository.<commit_after>import os
import sys
import multiprocessing
import subprocess
import signal
import urlparse
from yum.Errors import RepoError
from pakrat import util, log, repotools

__version__ = '0.0.7'

def sync(basedir, repos=[], repodirs=[], repofiles=[], repoversion=None, delete=False):
    util.validate_basedir(basedir)
    util.validate_repos(repos)
    util.validate_repofiles(repofiles)
    util.validate_repodirs(repodirs)
    util.validate_repos(repos)

    if repoversion:
        delete = False

    for file in repofiles:
        for filerepo in repotools.from_file(file):
            repos.append(filerepo)

    for dir in repodirs:
        for dirrepo in repotools.from_dir(dir):
            repos.append(dirrepo)

    processes = []
    for repo in repos:
        dest = util.get_repo_dir(basedir, repo.id)
        p = multiprocessing.Process(target=sync_repo, args=(repo, dest, repoversion, delete))
        p.start()
        processes.append(p)

    def cleanup(*args):
        for p in processes:
            os.kill(p.pid, signal.SIGKILL)
        sys.exit(1)

    signal.signal(signal.SIGINT, cleanup)
    signal.signal(signal.SIGTERM, cleanup)

    complete_count = 0
    while True:
        for p in processes:
            if not p.is_alive():
                complete_count += 1
        if complete_count == len(processes):
            break

def sync_repo(repo, dest, version, delete=False):
    try:
        packages_dir = util.get_packages_dir(dest)
        yb = util.get_yum()
        repo = repotools.set_path(repo, packages_dir)
        yb.repos.add(repo)
        yb.repos.enableRepo(repo.id)
        ygh = yb.doPackageLists(showdups=True)
        packages = ygh.available + ygh.reinstall_available
    except RepoError, e:
{'fraction_non_alphanumeric': 0.09385113268608414, 'fraction_numerical': 0.003236245954692557, 'mean_word_length': 2.750252780586451, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7176640', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1153, 'n_words': 278}