<commit_before>"""Configuration file for `buildercore`.

`buildercore.config` is a place to record assumptions really.
Settings that the users are encouraged to tweak should go into the
`settings.yml` file at the root of the project and incorporated here.

buildercore was originally part of the builder, then separated out
into it's own project 'builder-core' but has now been re-integrated.
This transition meant that `src/buildercore/` is still neatly separated
from the interface logic in the fabfile.

"""
import os
from os.path import join
from fabric.api import env
from buildercore import utils
from buildercore.utils import lmap, lfilter
from kids.cache import cache
import logging


# no un-catchable errors from Fabric

class FabricException(Exception):
    pass

env.abort_exception = FabricException

class ConfigurationError(Exception):
    pass

# dirs are relative
# paths are absolute
# a file is an absolute path to a file
# filenames are names of files without any other context


# these users should probably be specified in the project/org config file
# as 'defaults'. deploy user especially
ROOT_USER = 'root'
BOOTSTRAP_USER = 'ubuntu'
DEPLOY_USER = 'elife'

PROJECT_PATH = os.getcwd() # ll: /path/to/elife-builder/
SRC_PATH = join(PROJECT_PATH, 'src') # ll: /path/to/elife-builder/src/

TEMP_PATH = "/tmp/"

CFN = ".cfn"

STACK_DIR = join(CFN, "stacks") # ll: ./.cfn/stacks
CONTEXT_DIR = join(CFN, "contexts") # ll: ./.cfn/stacks
SCRIPTS_DIR = "scripts"
PRIVATE_DIR = "private"
KEYPAIR_DIR = join(CFN, "keypairs") # ll: ./.cfn/keypairs
# the .cfn dir was for cloudformation stuff, but we keep keypairs in there too, so this can't hurt
# perhaps a namechange from .cfn to .state or something later
TERRAFORM_DIR = join(CFN, "terraform")

STACK_PATH = join(PROJECT_PATH, STACK_DIR) # "/.../.cfn/stacks/"
CONTEXT_PATH = join(PROJECT_PATH, CONTEXT_DIR) # "/.../.cfn/contexts/"
KEYPAIR_PATH = join(PROJECT_PATH, KEYPAIR_DIR) # "/.../.cfn/keypairs/"
SCRIPTS_PATH = join(PROJECT_PATH, SCRIPTS_DIR) # "/.../scripts/"

# create all necessary paths and ensure they are writable
lmap(utils.mkdir_p, [TEMP_PATH, STACK_PATH, CONTEXT_PATH, SCRIPTS_PATH, KEYPAIR_PATH])

# logging

LOG_DIR = "logs"
LOG_PATH = join(PROJECT_PATH, LOG_DIR) # /.../logs/
LOG_FILE = join(LOG_PATH, "app.log") # /.../logs/app.log
utils.mkdir_p(LOG_PATH)

FORMAT = logging.Formatter("%(asctime)s - %(levelname)s - %(processName)s - %(name)s - %(message)s")
CONSOLE_FORMAT = logging.Formatter("%(levelname)s - %(name)s - %(message)s")

# http://docs.python.org/2/howto/logging-cookbook.html
ROOTLOG = logging.getLogger() # important! this is the *root LOG*
# all other LOGs are derived from this one
ROOTLOG.setLevel(logging.DEBUG) # *default* output level for all LOGs

# StreamHandler sends to stderr by default
CONSOLE_HANDLER = logging.StreamHandler()
CONSOLE_HANDLER.setLevel(logging.INFO) # output level for *this handler*
CONSOLE_HANDLER.setFormatter(CONSOLE_FORMAT)


# FileHandler sends to a named file
FILE_HANDLER = logging.FileHandler(LOG_FILE)
_log_level = os.environ.get('LOG_LEVEL_FILE', 'INFO')
FILE_HANDLER.setLevel(getattr(logging, _log_level))
FILE_HANDLER.setFormatter(FORMAT)

ROOTLOG.addHandler(CONSOLE_HANDLER)
ROOTLOG.addHandler(FILE_HANDLER)

LOG = logging.getLogger(__name__)
logging.getLogger('paramiko.transport').setLevel(logging.ERROR)
# TODO: leave on for FILE_HANDLER but not for CONSOLE_HANDLER
# logging.getLogger('botocore.vendored').setLevel(logging.ERROR)

def get_logger(name):
    "ensures logging is setup before handing out a Logger object to use"
    return logging.getLogger(name)

#
# remote
#

# where the builder can write stuff that should persist across installations/users
# like ec2 instance keypairs
BUILDER_BUCKET = 'elife-builder'
BUILDER_REGION = 'us-east-1'
BUILDER_NON_INTERACTIVE = 'BUILDER_NON_INTERACTIVE' in os.environ and os.environ['BUILDER_NON_INTERACTIVE']
BUILDER_TIMEOUT = 'BUILDER_TIMEOUT' in os.environ and int(os.environ['BUILDER_TIMEOUT'])
KEYPAIR_PREFIX = 'keypairs/'
CONTEXT_PREFIX = 'contexts/'

PACKER_BOX_PREFIX = "elifesciences" # the 'elifesciences' in 'elifesciences/basebox'
PACKER_BOX_BUCKET = "builder-boxes"
PACKER_BOX_KEY = "boxes"
# ll: s3://elife-builder/boxes
PACKER_BOX_S3_PATH = "s3://%s" % join(PACKER_BOX_BUCKET, PACKER_BOX_KEY)
PACKER_BOX_S3_HTTP_PATH = join("https://s3.amazonaws.com", PACKER_BOX_BUCKET, PACKER_BOX_KEY)

# these sections *shouldn't* be merged if they *don't* exist in the project
CLOUD_EXCLUDING_DEFAULTS_IF_NOT_PRESENT = ['rds', 'ext', 'elb', 'cloudfront', 'elasticache', 'fastly', 'eks']

#
# settings
# believe it or not but buildercore.config is NOT the place for user config
#
<commit_msg>Specify an high default BUILDER_TIMEOUT<commit_after>"""Configuration file for `buildercore`.

`buildercore.config` is a place to record assumptions really.
Settings that the users are encouraged to tweak should go into the
`settings.yml` file at the root of the project and incorporated here.

buildercore was originally part of the builder, then separated out
into it's own project 'builder-core' but has now been re-integrated.
This transition meant that `src/buildercore/` is still neatly separated
from the interface logic in the fabfile.

"""
import os
from os.path import join
from fabric.api import env
from buildercore import utils
from buildercore.utils import lmap, lfilter
from kids.cache import cache
import logging


# no un-catchable errors from Fabric

class FabricException(Exception):
    pass

env.abort_exception = FabricException

class ConfigurationError(Exception):
    pass

# dirs are relative
# paths are absolute
# a file is an absolute path to a file
# filenames are names of files without any other context


# these users should probably be specified in the project/org config file
# as 'defaults'. deploy user especially
ROOT_USER = 'root'
BOOTSTRAP_USER = 'ubuntu'
DEPLOY_USER = 'elife'

PROJECT_PATH = os.getcwd() # ll: /path/to/elife-builder/
SRC_PATH = join(PROJECT_PATH, 'src') # ll: /path/to/elife-builder/src/

TEMP_PATH = "/tmp/"

CFN = ".cfn"

STACK_DIR = join(CFN, "stacks") # ll: ./.cfn/stacks
CONTEXT_DIR = join(CFN, "contexts") # ll: ./.cfn/stacks
SCRIPTS_DIR = "scripts"
PRIVATE_DIR = "private"
KEYPAIR_DIR = join(CFN, "keypairs") # ll: ./.cfn/keypairs
# the .cfn dir was for cloudformation stuff, but we keep keypairs in there too, so this can't hurt
# perhaps a namechange from .cfn to .state or something later
TERRAFORM_DIR = join(CFN, "terraform")

STACK_PATH = join(PROJECT_PATH, STACK_DIR) # "/.../.cfn/stacks/"
CONTEXT_PATH = join(PROJECT_PATH, CONTEXT_DIR) # "/.../.cfn/contexts/"
KEYPAIR_PATH = join(PROJECT_PATH, KEYPAIR_DIR) # "/.../.cfn/keypairs/"
SCRIPTS_PATH = join(PROJECT_PATH, SCRIPTS_DIR) # "/.../scripts/"

# create all necessary paths and ensure they are writable
lmap(utils.mkdir_p, [TEMP_PATH, STACK_PATH, CONTEXT_PATH, SCRIPTS_PATH, KEYPAIR_PATH])

# logging

LOG_DIR = "logs"
LOG_PATH = join(PROJECT_PATH, LOG_DIR) # /.../logs/
LOG_FILE = join(LOG_PATH, "app.log") # /.../logs/app.log
utils.mkdir_p(LOG_PATH)

FORMAT = logging.Formatter("%(asctime)s - %(levelname)s - %(processName)s - %(name)s - %(message)s")
CONSOLE_FORMAT = logging.Formatter("%(levelname)s - %(name)s - %(message)s")

# http://docs.python.org/2/howto/logging-cookbook.html
ROOTLOG = logging.getLogger() # important! this is the *root LOG*
# all other LOGs are derived from this one
ROOTLOG.setLevel(logging.DEBUG) # *default* output level for all LOGs

# StreamHandler sends to stderr by default
CONSOLE_HANDLER = logging.StreamHandler()
CONSOLE_HANDLER.setLevel(logging.INFO) # output level for *this handler*
CONSOLE_HANDLER.setFormatter(CONSOLE_FORMAT)


# FileHandler sends to a named file
FILE_HANDLER = logging.FileHandler(LOG_FILE)
_log_level = os.environ.get('LOG_LEVEL_FILE', 'INFO')
FILE_HANDLER.setLevel(getattr(logging, _log_level))
FILE_HANDLER.setFormatter(FORMAT)

ROOTLOG.addHandler(CONSOLE_HANDLER)
ROOTLOG.addHandler(FILE_HANDLER)

LOG = logging.getLogger(__name__)
logging.getLogger('paramiko.transport').setLevel(logging.ERROR)
# TODO: leave on for FILE_HANDLER but not for CONSOLE_HANDLER
# logging.getLogger('botocore.vendored').setLevel(logging.ERROR)

def get_logger(name):
    "ensures logging is setup before handing out a Logger object to use"
    return logging.getLogger(name)

#
# remote
#

# where the builder can write stuff that should persist across installations/users
# like ec2 instance keypairs
BUILDER_BUCKET = 'elife-builder'
BUILDER_REGION = 'us-east-1'
BUILDER_NON_INTERACTIVE = 'BUILDER_NON_INTERACTIVE' in os.environ and os.environ['BUILDER_NON_INTERACTIVE']
if 'BUILDER_TIMEOUT' in os.environ:
    BUILDER_TIMEOUT = int(os.environ['BUILDER_TIMEOUT'])
else:
    BUILDER_TIMEOUT = 600
KEYPAIR_PREFIX = 'keypairs/'
CONTEXT_PREFIX = 'contexts/'

PACKER_BOX_PREFIX = "elifesciences" # the 'elifesciences' in 'elifesciences/basebox'
PACKER_BOX_BUCKET = "builder-boxes"
PACKER_BOX_KEY = "boxes"
# ll: s3://elife-builder/boxes
PACKER_BOX_S3_PATH = "s3://%s" % join(PACKER_BOX_BUCKET, PACKER_BOX_KEY)
PACKER_BOX_S3_HTTP_PATH = join("https://s3.amazonaws.com", PACKER_BOX_BUCKET, PACKER_BOX_KEY)

# these sections *shouldn't* be merged if they *don't* exist in the project
CLOUD_EXCLUDING_DEFAULTS_IF_NOT_PRESENT = ['rds', 'ext', 'elb', 'cloudfront', 'elasticache', 'fastly', 'eks']

#
# settings
# believe it or not but buildercore.config is NOT the place for user config
#

{'fraction_non_alphanumeric': 0.1128177966101695, 'fraction_numerical': 0.0020127118644067796, 'mean_word_length': 5.281437125748503, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3796777', 'n_tokens_mistral': 3241, 'n_tokens_neox': 3262, 'n_words': 897}