<commit_before>#!/usr/bin/env python
#
# Copyright 2015 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4:

from __future__ import (absolute_import, division, print_function,
                        with_statement)

import importlib
import yaml
import os
import logging

# Setting root path
ROOT = None
if os.path.abspath(__file__).endswith('.py') or \
        os.path.abspath(__file__).endswith('.pyc'):
    ROOT = os.path.dirname(os.path.abspath(__file__))
else:
    ROOT = os.path.abspath(__file__)
ROOT = os.path.abspath(os.path.join(ROOT, '..'))

# Getting configuration paths and files from the environment
FIRENADO_CONFIG_FILE = None
try:
    FIRENADO_CONFIG_FILE = os.environ['FIRENADO_CONFIG_FILE']
except KeyError:
    FIRENADO_CONFIG_FILE = 'firenado.yaml'

stack = []

LIB_CONFIG_FILE = os.path.join(ROOT, 'conf', FIRENADO_CONFIG_FILE)

# Application file
APP_ROOT_PATH = os.path.join(os.getcwd())
# If FIRENADO_CURRENT_APP_PATH is not set than return current directory
# conf dir
APP_CONFIG_PATH = os.getenv('FIRENADO_CURRENT_APP_CONFIG_PATH',
                            os.path.join(APP_ROOT_PATH, 'conf'))
APP_CONFIG_FILE = os.path.join(APP_CONFIG_PATH, FIRENADO_CONFIG_FILE)

HAS_LIB_CONFIG_FILE = False
HAS_APP_CONFIG_FILE = False

if os.path.isfile(LIB_CONFIG_FILE):
    HAS_LIB_CONFIG_FILE = True
    stack.append(LIB_CONFIG_FILE)

if os.path.isfile(APP_CONFIG_FILE):
    if APP_CONFIG_FILE != LIB_CONFIG_FILE:
        HAS_APP_CONFIG_FILE = True
        stack.append(APP_CONFIG_FILE)

# Tmp path variable<commit_msg>Changed conf to read firenado.yml files too.<commit_after>#!/usr/bin/env python
#
# Copyright 2015 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4:

from __future__ import (absolute_import, division, print_function,
                        with_statement)

import importlib
import yaml
import os
import logging

# Setting root path
ROOT = None
if os.path.abspath(__file__).endswith('.py') or \
        os.path.abspath(__file__).endswith('.pyc'):
    ROOT = os.path.dirname(os.path.abspath(__file__))
else:
    ROOT = os.path.abspath(__file__)
ROOT = os.path.abspath(os.path.join(ROOT, '..'))

# Getting configuration paths and files from the environment
FIRENADO_CONFIG_FILE = None
try:
    FIRENADO_CONFIG_FILE = os.environ['FIRENADO_CONFIG_FILE']
except KeyError:
    FIRENADO_CONFIG_FILE = 'firenado'

stack = []

LIB_CONFIG_FILE = os.path.join(ROOT, 'conf', FIRENADO_CONFIG_FILE)

# Application file
APP_ROOT_PATH = os.path.join(os.getcwd())
# If FIRENADO_CURRENT_APP_PATH is not set than return current directory
# conf dir
APP_CONFIG_PATH = os.getenv('FIRENADO_CURRENT_APP_CONFIG_PATH',
                            os.path.join(APP_ROOT_PATH, 'conf'))
APP_CONFIG_FILE = os.path.join(APP_CONFIG_PATH, FIRENADO_CONFIG_FILE)

HAS_LIB_CONFIG_FILE = False
HAS_APP_CONFIG_FILE = False

config_file_extensions = ['yml', 'yaml']
for extension in config_file_extensions:
    if not HAS_LIB_CONFIG_FILE:
        if os.path.isfile('%s.%s' % (LIB_CONFIG_FILE, extension)):
            HAS_LIB_CONFIG_FILE = True
            LIB_CONFIG_FILE = '%s.%s' % (LIB_CONFIG_FILE, extension)
            stack.append(LIB_CONFIG_FILE)
    if not HAS_APP_CONFIG_FILE:
        if os.path.isfile('%s.%s' % (APP_CONFIG_FILE, extension)):
            HAS_APP_CONFIG_FILE = True
            APP_CONFIG_FILE = '%s.%s' % (APP_CONFIG_FILE, extension)
            stack.append(APP_CONFIG_FILE)

# Tmp path variable
{'fraction_non_alphanumeric': 0.09564643799472296, 'fraction_numerical': 0.004837291116974494, 'mean_word_length': 4.048834628190899, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '207596', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1451, 'n_words': 395}