<commit_before>#!/usr/bin/env python
# A script to keep our git repos up to date

import os, time, shutil, subprocess
from koji import ClientSession
from fedora.client.pkgdb import PackageDB

REPO_DIR = '/var/cache/fedoracommunity/git.fedoraproject.org'
TIMESTAMP = '/var/cache/fedoracommunity/git.fedoraproject.org/.timestamp'

# Get a list of active git branches
pkgdb = PackageDB()
collections = pkgdb.get_collection_list(eol=False)
active = [c[0].gitbranchname for c in collections]

# Grab a list of all koji builds since our last run
# if there is no saved timestamp, do a full run
packages = []
if not os.path.exists(TIMESTAMP):
    packages = os.listdir(REPO_DIR)
else:
    timestamp = file(TIMESTAMP).read().strip()
    koji = ClientSession('http://koji.fedoraproject.org/kojihub')
    builds = koji.listBuilds(queryOpts={'createdAfter': float(timestamp)})
    packages = set([build['name'] for build in builds])
    packages = [pkg for pkg in packages
                if os.path.isdir(os.path.join(REPO_DIR, pkg))]

for repo in packages:
    print("[ %s ]" % repo)
    for branch in os.listdir(os.path.join(REPO_DIR, repo)):
        if branch in active:<commit_msg>Fix our koji.listBuilds queryOpts<commit_after>#!/usr/bin/env python
# A script to keep our git repos up to date

import os, time, shutil, subprocess
from koji import ClientSession
from fedora.client.pkgdb import PackageDB

REPO_DIR = '/var/cache/fedoracommunity/git.fedoraproject.org'
TIMESTAMP = '/var/cache/fedoracommunity/git.fedoraproject.org/.timestamp'

# Get a list of active git branches
pkgdb = PackageDB()
collections = pkgdb.get_collection_list(eol=False)
active = [c[0].gitbranchname for c in collections]

# Grab a list of all koji builds since our last run
# if there is no saved timestamp, do a full run
packages = []
if not os.path.exists(TIMESTAMP):
    packages = os.listdir(REPO_DIR)
else:
    timestamp = file(TIMESTAMP).read().strip()
    koji = ClientSession('http://koji.fedoraproject.org/kojihub')
    builds = koji.listBuilds(createdAfter=float(timestamp))
    packages = set([build['name'] for build in builds])
    packages = [pkg for pkg in packages
                if os.path.isdir(os.path.join(REPO_DIR, pkg))]

for repo in packages:
    print("[ %s ]" % repo)
    for branch in os.listdir(os.path.join(REPO_DIR, repo)):
        if branch in active:
{'fraction_non_alphanumeric': 0.11464968152866242, 'fraction_numerical': 0.0008492569002123143, 'mean_word_length': 4.077586206896552, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1530912', 'n_tokens_mistral': 790, 'n_tokens_neox': 791, 'n_words': 225}