<commit_before>#
# Copyright European Organization for Nuclear Research (CERN),
#           National Research Centre "Kurchatov Institute" (NRC KI)
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - <NAME>, <<EMAIL>>, 2017
#

from datetime import datetime as dt

from ..utils import EnumTypes

HDFS_BASE_DIR = '/atlas/prodsys_model'

HDFS_DATA_DIR = '{0}/data'.format(HDFS_BASE_DIR)
HDFS_PRIVATE_DIR = '{0}/private'.format(HDFS_BASE_DIR)

STORAGE_PATH_FORMAT = 'hdfs://{0}/{1}'.format(HDFS_DATA_DIR, '{dir_name}')

WORK_DIR_NAME_DEFAULT = dt.utcnow().strftime('%Y%m%d')

DataType = EnumTypes(
    ('Training', 'training'),
    ('Test', 'test'),
    ('Model', 'model'),
    ('Eval', 'eval'),
    ('Input', 'input'),
    ('Output', 'output'),
)

DirType = EnumTypes(
    ('Work', 'work'),
    ('Data', 'data')
)


try:
    from .settings_local import *
except ImportError:
    pass<commit_msg>added new data-type "domain" that describes the range of possible values (used as a reference for labeled_points data)<commit_after>#
# Copyright European Organization for Nuclear Research (CERN),
#           National Research Centre "Kurchatov Institute" (NRC KI)
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - <NAME>, <<EMAIL>>, 2017
#

from datetime import datetime as dt

from ..utils import EnumTypes

HDFS_BASE_DIR = '/atlas/prodsys_model'

HDFS_DATA_DIR = '{0}/data'.format(HDFS_BASE_DIR)
HDFS_PRIVATE_DIR = '{0}/private'.format(HDFS_BASE_DIR)

STORAGE_PATH_FORMAT = 'hdfs://{0}/{1}'.format(HDFS_DATA_DIR, '{dir_name}')

WORK_DIR_NAME_DEFAULT = dt.utcnow().strftime('%Y%m%d')

DataType = EnumTypes(
    ('Training', 'training'),
    ('Test', 'test'),
    ('Model', 'model'),
    ('Eval', 'eval'),
    ('Input', 'input'),
    ('Output', 'output'),
    ('Domain', 'domain'),
)

DirType = EnumTypes(
    ('Work', 'work'),
    ('Data', 'data')
)


try:
    from .settings_local import *
except ImportError:
    pass
{'fraction_non_alphanumeric': 0.16123424598000868, 'fraction_numerical': 0.010430247718383311, 'mean_word_length': 4.1730337078651685, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5753971', 'n_tokens_mistral': 810, 'n_tokens_neox': 778, 'n_words': 214}