<commit_before>#!/usr/bin/env python
# encoding: utf-8

'''
Automatically create jenkins jobs for the refs in a git repository.
Documentation: http://gvalkov.github.com/jenkins-autojobs/
'''

import re

from os import linesep, path
from sys import exit, argv

from lxml import etree
from jenkins_autojobs.main import main as _main, debug_refconfig
from jenkins_autojobs.util import sanitize, check_output, merge
from jenkins_autojobs.job import Job

import time
import datetime


#-----------------------------------------------------------------------------
def git_refs_iter_local(repo, *args):
    cmd = ('git', 'show-ref')
    out = check_output(cmd, cwd=repo).split(linesep)

    return (ref for sha, ref in [i.split() for i in out if i])

def git_refs_iter_remote(repo, stale, before, yamlpath):
    cmd = ('git', 'ls-remote', '--heads', repo)
    out = check_output(cmd).decode('utf8').split(linesep)

    for sha, ref in (i.split() for i in out if i):
        if not ref.startswith('refs/'):
            continue
        # :todo: generalize
        if ref.endswith('^{}'):
            continue

        if stale and stale_branch(ref, repo, stale, before, yamlpath):
            continue

        yield ref

# Takes a string argument `ref` that corresponds
# to a branch in `repo`
# Returns the timestamp as shown by
# --date=short
def get_newest_commit_from_branch(ref, repo, yamlpath):
    repo_path = path.join(path.dirname(path.realpath(yamlpath)), 'cassandra')
    if not path.isdir(repo_path):
        cmd = ('git', 'clone', '--bare', 'https://github.com/apache/cassandra', repo_path)
        check_output(cmd).decode('utf8')
    cmd = ('git', '--git-dir=%s' % repo_path, 'fetch', repo, ref)
    check_output(cmd).decode('utf8')
    cmd = ('git', '--git-dir=%s' % repo_path, 'show', '--date=short', '--quiet', 'FETCH_HEAD')
    out = check_output(cmd).decode('utf8')
    return out

# Takes the timestamp for a git commit
# In the git --date=short format.
# Returns the unix timestamp.
def get_commit_timestamp(message):
    commit_date = re.search('Date:(.*?)\n', message).groups()[0].strip()
    return time.mktime(datetime.datetime.strptime(commit_date, "%Y-%m-%d").timetuple())

# Takes a string argument `ref` that corresponds
# to a branch in `repo`
# Returns if the branch is more than 3 months old
# Or if the branch is from before we started the autojobs
# project. This is a simple timestamp check.
# Easiest way to do it statelessly.
def stale_branch(ref, repo, stale, before, yamlpath):<commit_msg>Use commit date, not author date<commit_after>#!/usr/bin/env python
# encoding: utf-8

'''
Automatically create jenkins jobs for the refs in a git repository.
Documentation: http://gvalkov.github.com/jenkins-autojobs/
'''

import re

from os import linesep, path
from sys import exit, argv

from lxml import etree
from jenkins_autojobs.main import main as _main, debug_refconfig
from jenkins_autojobs.util import sanitize, check_output, merge
from jenkins_autojobs.job import Job

import time
import datetime


#-----------------------------------------------------------------------------
def git_refs_iter_local(repo, *args):
    cmd = ('git', 'show-ref')
    out = check_output(cmd, cwd=repo).split(linesep)

    return (ref for sha, ref in [i.split() for i in out if i])

def git_refs_iter_remote(repo, stale, before, yamlpath):
    cmd = ('git', 'ls-remote', '--heads', repo)
    out = check_output(cmd).decode('utf8').split(linesep)

    for sha, ref in (i.split() for i in out if i):
        if not ref.startswith('refs/'):
            continue
        # :todo: generalize
        if ref.endswith('^{}'):
            continue

        if stale and stale_branch(ref, repo, stale, before, yamlpath):
            continue

        yield ref

# Takes a string argument `ref` that corresponds
# to a branch in `repo`
# Returns the timestamp as shown by
# --date=short
def get_newest_commit_from_branch(ref, repo, yamlpath):
    repo_path = path.join(path.dirname(path.realpath(yamlpath)), 'cassandra')
    if not path.isdir(repo_path):
        cmd = ('git', 'clone', '--bare', 'https://github.com/apache/cassandra', repo_path)
        check_output(cmd).decode('utf8')
    cmd = ('git', '--git-dir=%s' % repo_path, 'fetch', repo, ref)
    check_output(cmd).decode('utf8')
    cmd = ('git', '--git-dir=%s' % repo_path, 'show', '--format=%cd','--date=short', '--quiet', 'FETCH_HEAD')
    out = check_output(cmd).decode('utf8')
    return out

# Takes the timestamp for a git commit
# In the git --date=short format.
# Returns the unix timestamp.
def get_commit_timestamp(message):
    commit_date = message.strip()
    return time.mktime(datetime.datetime.strptime(commit_date, "%Y-%m-%d").timetuple())

# Takes a string argument `ref` that corresponds
# to a branch in `repo`
# Returns if the branch is more than 3 months old
# Or if the branch is from before we started the autojobs
# project. This is a simple timestamp check.
# Easiest way to do it statelessly.
def stale_branch(ref, repo, stale, before, yamlpath):
{'fraction_non_alphanumeric': 0.15949216425312437, 'fraction_numerical': 0.002578853402102757, 'mean_word_length': 3.2263202011735124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4002559', 'n_tokens_mistral': 1636, 'n_tokens_neox': 1582, 'n_words': 505}