<commit_before>#!/usr/bin/env python3
# Copyright (c) 2017 SUSE LLC

import os
import logging
import shutil
import sys
import time

from stagelib.envglobals import (verifyRequiredEnvVars, exportGitInfoEnvVars, getEnvVar)
from stagelib.filetools import (list_files, mkdir_if_dne, copy_files, recursive_copy_dir,
                                IOOSErrorGracefulFail, save_files_copied)
import stagelib.git as git
from stagelib.replace import do_variable_replace
from stagelib.blacklist import get_blacklist


# Set default values for tunables (primarily only interesting for testing)
CORE_FILES_DIR = "src"
CEPH_RELEASES_DIR = "ceph-releases"
BLACKLIST_FILE = "flavor-blacklist.txt"


STAGING_DIR = getEnvVar('STAGING_DIR')
# Start with empty staging dir so there are no previous artifacts
try:
    if os.path.isdir(STAGING_DIR):
        shutil.rmtree(STAGING_DIR)
    os.makedirs(STAGING_DIR, mode=0o755)
except (OSError, IOError) as o:
    IOOSErrorGracefulFail(
        o, 'Could not delete and recreate staging dir: {}'.format(STAGING_DIR))


LOG_FILE = os.path.join(STAGING_DIR, "stage.log")
loglevel = logging.INFO
# If DEBUG env var is set to anything (including empty string) except '0', log debug text
if 'DEBUG' in os.environ and not os.environ['DEBUG'] == '0':
    loglevel = logging.DEBUG
logging.basicConfig(filename=LOG_FILE, level=loglevel, format='%(levelname)5s:  %(message)s')

# Build dependency on python3 for `replace.py`. Looking to py2.7 deprecation in 2020.
if sys.version_info[0] < 3:
    print('This must be run with Python 3+')
    sys.exit(1)


def main(CORE_FILES_DIR, CEPH_RELEASES_DIR, BLACKLIST_FILE):
    logging.info('\n\n\n')  # Make it easier to determine where new runs start
    logging.info('Start time: {}'.format(time.ctime()))
    logging.info('Git repo:   {}'.format(git.get_repo()))
    logging.info('Git branch: {}'.format(git.get_branch()))
    logging.info('Git commit: {}'.format(git.get_hash()))

    print('')
    verifyRequiredEnvVars()
    print('')
    exportGitInfoEnvVars()

    CEPH_VERSION = getEnvVar('CEPH_VERSION')
    OS_NAME = getEnvVar('OS_NAME')
    OS_VERSION = getEnvVar('OS_VERSION')
    IMAGES_TO_BUILD = getEnvVar('IMAGES_TO_BUILD').split(' ')
    # STAGING_DIR is gotten globally

    # Search from least specfic to most specific
    path_search_order = [
        "{}".format(CORE_FILES_DIR),
        os.path.join(CEPH_RELEASES_DIR, 'ALL'),
        os.path.join(CEPH_RELEASES_DIR, 'ALL', OS_NAME),
        os.path.join(CEPH_RELEASES_DIR, 'ALL', OS_NAME, OS_VERSION),
        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION),
        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, OS_NAME),
        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, OS_NAME, OS_VERSION),<commit_msg>staging: log git env vars<commit_after>#!/usr/bin/env python3
# Copyright (c) 2017 SUSE LLC

import os
import logging
import shutil
import sys
import time

from stagelib.envglobals import (verifyRequiredEnvVars, exportGitInfoEnvVars, getEnvVar)
from stagelib.filetools import (list_files, mkdir_if_dne, copy_files, recursive_copy_dir,
                                IOOSErrorGracefulFail, save_files_copied)
from stagelib.replace import do_variable_replace
from stagelib.blacklist import get_blacklist


# Set default values for tunables (primarily only interesting for testing)
CORE_FILES_DIR = "src"
CEPH_RELEASES_DIR = "ceph-releases"
BLACKLIST_FILE = "flavor-blacklist.txt"


STAGING_DIR = getEnvVar('STAGING_DIR')
# Start with empty staging dir so there are no previous artifacts
try:
    if os.path.isdir(STAGING_DIR):
        shutil.rmtree(STAGING_DIR)
    os.makedirs(STAGING_DIR, mode=0o755)
except (OSError, IOError) as o:
    IOOSErrorGracefulFail(
        o, 'Could not delete and recreate staging dir: {}'.format(STAGING_DIR))


LOG_FILE = os.path.join(STAGING_DIR, "stage.log")
loglevel = logging.INFO
# If DEBUG env var is set to anything (including empty string) except '0', log debug text
if 'DEBUG' in os.environ and not os.environ['DEBUG'] == '0':
    loglevel = logging.DEBUG
logging.basicConfig(filename=LOG_FILE, level=loglevel, format='%(levelname)5s:  %(message)s')

# Build dependency on python3 for `replace.py`. Looking to py2.7 deprecation in 2020.
if sys.version_info[0] < 3:
    print('This must be run with Python 3+')
    sys.exit(1)


def main(CORE_FILES_DIR, CEPH_RELEASES_DIR, BLACKLIST_FILE):
    logging.info('\n\n\n')  # Make it easier to determine where new runs start
    logging.info('Start time: {}'.format(time.ctime()))

    print('')
    verifyRequiredEnvVars()
    print('')

    exportGitInfoEnvVars()
    logging.info('GIT_REPO:    {}'.format(getEnvVar('GIT_REPO')))
    logging.info('GIT_BRANCH: {}'.format(getEnvVar('GIT_BRANCH')))
    logging.info('GIT_COMMIT: {}'.format(getEnvVar('GIT_COMMIT')))
    logging.info('GIT_CLEAN:  {}'.format(getEnvVar('GIT_CLEAN')))

    CEPH_VERSION = getEnvVar('CEPH_VERSION')
    OS_NAME = getEnvVar('OS_NAME')
    OS_VERSION = getEnvVar('OS_VERSION')
    IMAGES_TO_BUILD = getEnvVar('IMAGES_TO_BUILD').split(' ')
    # STAGING_DIR is gotten globally

    # Search from least specfic to most specific
    path_search_order = [
        "{}".format(CORE_FILES_DIR),
        os.path.join(CEPH_RELEASES_DIR, 'ALL'),
        os.path.join(CEPH_RELEASES_DIR, 'ALL', OS_NAME),
        os.path.join(CEPH_RELEASES_DIR, 'ALL', OS_NAME, OS_VERSION),
        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION),
        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, OS_NAME),
        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, OS_NAME, OS_VERSION),
{'fraction_non_alphanumeric': 0.1158273381294964, 'fraction_numerical': 0.008273381294964029, 'mean_word_length': 4.383349467570184, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024499', 'n_tokens_mistral': 1996, 'n_tokens_neox': 1941, 'n_words': 440}