<commit_before>#!/usr/bin/python

import argparse
import re
import time
from datetime import datetime

import requests
import urllib3
import yaml
from diskcache import Cache

urllib3.disable_warnings()

try:
    from urlparse import urljoin
except ImportError:
    from urllib.parse import urljoin


OOO_PROJECTS = [
    'openstack/puppet-tripleo', 'openstack/python-tripleoclient',
    'openstack/tripleo-upgrade', 'openstack/tripleo-quickstart-extras',
    'openstack/tripleo-common', 'openstack/tripleo-ci',
    'openstack/tripleo-quickstart', 'openstack/tripleo-heat-templates',
    'openstack/tripleo-ansible', 'openstack/tripleo-validations',
    'rdo-infra/ansible-role-tripleo-ci-reproducer',
    'containers/libpod', 'ceph/ceph-ansible',
    'openstack/ansible-collections-openstack', 'openstack/heat',
    'rdoinfo', 'nfvinfo'
]

INTERNAL_OOO_PROJECTS = [
    'openstack/tripleo-ci-internal-jobs',
    'rhos-release',
    'openstack-tempest',
    'openstack-tripleo-common',
    'openstack-tripleo-heat-templates-compat',
    'openstack-tripleo-heat-templates',
    'openstack-tripleo-image-elements',
    'openstack-tripleo-puppet-elements',
    'openstack-tripleo-ui',
    'openstack-tripleo-validations',
    'openstack-tripleo',
    'openstack/tripleo-ci',
    'rhos-ops/openstack-tripleo-heat-templates'
]

TIMESTAMP_PATTERN = '%Y-%m-%dT%H:%M:%S'
TIMESTAMP_PATTERN2 = '%Y-%m-%d %H:%M:%S'

JOBS_FOR_ARA = []
#    'tripleo-ci-centos-7-standalone',
#    'tripleo-ci-centos-7-containers-multinode',
#    'tripleo-ci-centos-7-undercloud-containers',
#    'tripleo-ci-centos-7-scenario001-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario002-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario003-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario004-multinode-oooq-container',
# ]

ARA_JSONS = [
    '/logs/ara.oooq.root.json',
    '/logs/ara.oooq.oc.json',
    '/logs/ara.json'
]

TASK_DURATION_TRESHOLD = 10
CACHE_EXPIRE_SECS = 604800  # a week
CACHE_SIZE = int(2e9)  # 2GB

cache = Cache('/tmp/ruck_rover_cache', size_limit=CACHE_SIZE)
cache.expire()

# Convert datetime to timestamp


def to_ts(d, seconds=False, pattern=TIMESTAMP_PATTERN):
    return datetime.strptime(
        d, pattern).strftime('%s') + ('' if seconds else "000000000")


def to_seconds(duration):
    x = time.strptime(duration, '%H:%M:%S')
    return datetime.timedelta(
        hours=x.tm_hour, minutes=x.tm_min, seconds=x.tm_sec).total_seconds()


def get(url, json_view, query=None, timeout=20):
    query = query or {}
    try:
        response = requests.get(url,
                                params=query,
                                timeout=timeout,
                                verify=False)
        if response and response.ok:
            if json_view:
                return response.json()
            else:
                response.encoding = 'utf-8'
                return response.text
    except Exception:
        return None

    return None


def get_builds_info(url, query, pages, offset):
    builds = []
    for p in range(pages):
        if p > 0:
            query['skip'] = offset + (p * 50)
            # let's not abuse ZUUL API and sleep betwen requests
            time.sleep(2)
        else:
            query['skip'] = offset
        builds_api = url + "builds" + '?complete=true'
        response = get(builds_api, True, query)
        if response is not None:
            builds += response<commit_msg>update api build url for rdo zuul builds<commit_after>#!/usr/bin/python

import argparse
import re
import time
from datetime import datetime

import requests
import urllib3
import yaml
from diskcache import Cache

urllib3.disable_warnings()

try:
    from urlparse import urljoin
except ImportError:
    from urllib.parse import urljoin


OOO_PROJECTS = [
    'openstack/puppet-tripleo', 'openstack/python-tripleoclient',
    'openstack/tripleo-upgrade', 'openstack/tripleo-quickstart-extras',
    'openstack/tripleo-common', 'openstack/tripleo-ci',
    'openstack/tripleo-quickstart', 'openstack/tripleo-heat-templates',
    'openstack/tripleo-ansible', 'openstack/tripleo-validations',
    'rdo-infra/ansible-role-tripleo-ci-reproducer',
    'containers/libpod', 'ceph/ceph-ansible',
    'openstack/ansible-collections-openstack', 'openstack/heat',
    'rdoinfo', 'nfvinfo'
]

INTERNAL_OOO_PROJECTS = [
    'openstack/tripleo-ci-internal-jobs',
    'rhos-release',
    'openstack-tempest',
    'openstack-tripleo-common',
    'openstack-tripleo-heat-templates-compat',
    'openstack-tripleo-heat-templates',
    'openstack-tripleo-image-elements',
    'openstack-tripleo-puppet-elements',
    'openstack-tripleo-ui',
    'openstack-tripleo-validations',
    'openstack-tripleo',
    'openstack/tripleo-ci',
    'rhos-ops/openstack-tripleo-heat-templates'
]

TIMESTAMP_PATTERN = '%Y-%m-%dT%H:%M:%S'
TIMESTAMP_PATTERN2 = '%Y-%m-%d %H:%M:%S'

JOBS_FOR_ARA = []
#    'tripleo-ci-centos-7-standalone',
#    'tripleo-ci-centos-7-containers-multinode',
#    'tripleo-ci-centos-7-undercloud-containers',
#    'tripleo-ci-centos-7-scenario001-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario002-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario003-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario004-multinode-oooq-container',
# ]

ARA_JSONS = [
    '/logs/ara.oooq.root.json',
    '/logs/ara.oooq.oc.json',
    '/logs/ara.json'
]

TASK_DURATION_TRESHOLD = 10
CACHE_EXPIRE_SECS = 604800  # a week
CACHE_SIZE = int(2e9)  # 2GB

cache = Cache('/tmp/ruck_rover_cache', size_limit=CACHE_SIZE)
cache.expire()

# Convert datetime to timestamp


def to_ts(d, seconds=False, pattern=TIMESTAMP_PATTERN):
    return datetime.strptime(
        d, pattern).strftime('%s') + ('' if seconds else "000000000")


def to_seconds(duration):
    x = time.strptime(duration, '%H:%M:%S')
    return datetime.timedelta(
        hours=x.tm_hour, minutes=x.tm_min, seconds=x.tm_sec).total_seconds()


def get(url, json_view, query=None, timeout=20):
    query = query or {}
    try:
        response = requests.get(url,
                                params=query,
                                timeout=timeout,
                                verify=False)
        if response and response.ok:
            if json_view:
                return response.json()
            else:
                response.encoding = 'utf-8'
                return response.text
    except Exception:
        return None

    return None


def get_builds_info(url, query, pages, offset):
    builds = []
    for p in range(pages):
        if p > 0:
            query['skip'] = offset + (p * 50)
            # let's not abuse ZUUL API and sleep betwen requests
            time.sleep(2)
        else:
            query['skip'] = offset
        if "rdo" in url:
            builds_api = url + "builds"
        else:
            builds_api = url + "builds" + '?complete=true'
        response = get(builds_api, True, query)
        if response is not None:
            builds += response
{'fraction_non_alphanumeric': 0.13210227272727273, 'fraction_numerical': 0.013920454545454545, 'mean_word_length': 2.8307943416757344, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6892161', 'n_tokens_mistral': 2594, 'n_tokens_neox': 2468, 'n_words': 412}