<commit_before>"""Utilities to track and record file system."""

import os
import uuid
import shutil
import json
import re
import six

STUDIOML_EXPERIMENT = 'STUDIOML_EXPERIMENT'
STUDIOML_HOME = 'STUDIOML_HOME'
STUDIOML_ARTIFACT_MAPPING = 'STUDIOML_ARTIFACT_MAPPING'


def get_experiment_key():
    if STUDIOML_EXPERIMENT not in os.environ.keys():
        key = str(<KEY>())
        setup_experiment(os.environ, key)
    return os.environ[STUDIOML_EXPERIMENT]


def get_studio_home():
    if STUDIOML_HOME in os.environ.keys():
        return os.environ[STUDIOML_HOME]
    else:
        return os.path.join(os.path.expanduser('~'), '.studioml')


def setup_experiment(env, experiment, clean=True):
    if not isinstance(experiment, str):
        key = experiment.key
        artifacts = experiment.artifacts
    else:
        key = experiment
        artifacts = {}

    env[STUDIOML_EXPERIMENT] = key
    _setup_model_directory(key, clean)

    artifact_mapping_path = _get_artifact_mapping_path(key)
    env[STUDIOML_ARTIFACT_MAPPING] = artifact_mapping_path

    amapping = {}
    for tag, art in six.iteritems(artifacts):
        if art.get('local') is not None:
            amapping[tag] = art['local']

        with open(artifact_mapping_path, 'w') as f:
            json.dump(amapping, f)


def get_artifact(tag):
    try:
        mapping_path = os.environ.get(STUDIOML_ARTIFACT_MAPPING)
        if mapping_path:
            with open(mapping_path, 'r') as f:
                a_mapping = json.load(f)
            return a_mapping[tag]
        else:
            return os.path.join(os.getcwd(), '..', tag)
    except BaseException:
        return None


def get_artifacts():
    try:
        with open(_get_artifact_mapping_path(), 'r') as f:
            return json.load(f)
    except BaseException:
        return {}


def get_model_directory(experiment_name=None):
    return get_artifact_cache('modeldir', experiment_name)


def get_artifact_cache(tag, experiment_name=None):
    assert tag is not None

    if tag.startswith('experiments/'):
        experiment_name = re.sub(
            '\Aexperiments/',
            '',
            re.sub(
                '/[^/]*\Z',
                '',
                tag))
        tag = re.sub('\.tar\.?[^\.]*\Z', '', re.sub('.*/', '', tag))

    if tag.startswith('blobstore/'):
        return get_blob_cache(tag)

    experiment_name = experiment_name if experiment_name else \
        get_experiment_key()<commit_msg>implementent fs_tracker.get_artifacts method with default artifact mapping<commit_after>"""Utilities to track and record file system."""

import os
import uuid
import shutil
import json
import re
import six

STUDIOML_EXPERIMENT = 'STUDIOML_EXPERIMENT'
STUDIOML_HOME = 'STUDIOML_HOME'
STUDIOML_ARTIFACT_MAPPING = 'STUDIOML_ARTIFACT_MAPPING'


def get_experiment_key():
    if STUDIOML_EXPERIMENT not in os.environ.keys():
        key = str(uuid.uuid4())
        setup_experiment(os.environ, key)
    return os.environ[STUDIOML_EXPERIMENT]


def get_studio_home():
    if STUDIOML_HOME in os.environ.keys():
        return os.environ[STUDIOML_HOME]
    else:
        return os.path.join(os.path.expanduser('~'), '.studioml')


def setup_experiment(env, experiment, clean=True):
    if not isinstance(experiment, str):
        key = experiment.key
        artifacts = experiment.artifacts
    else:
        key = experiment
        artifacts = {}

    env[STUDIOML_EXPERIMENT] = key
    _setup_model_directory(key, clean)

    artifact_mapping_path = _get_artifact_mapping_path(key)
    env[STUDIOML_ARTIFACT_MAPPING] = artifact_mapping_path

    amapping = {}
    for tag, art in six.iteritems(artifacts):
        if art.get('local') is not None:
            amapping[tag] = art['local']

        with open(artifact_mapping_path, 'w') as f:
            json.dump(amapping, f)


def get_artifact(tag):
    try:
        mapping_path = os.environ.get(STUDIOML_ARTIFACT_MAPPING)
        if mapping_path:
            with open(mapping_path, 'r') as f:
                a_mapping = json.load(f)
            return a_mapping[tag]
        else:
            return os.path.join(os.getcwd(), '..', tag)
    except BaseException:
        return None


def get_artifacts():
    try:
        mapping_path = os.environ.get(STUDIOML_ARTIFACT_MAPPING)
        if mapping_path:
            with open(mapping_path, 'r') as f:
                return json.load(f)
        else:
            artifacts = os.listdir(os.path.join(os.getcwd(), '..'))
            return {art: os.path.join(os.getcwd(), '..', art)
                    for art in artifacts}
    except BaseException:
        return {}


def get_model_directory(experiment_name=None):
    return get_artifact_cache('modeldir', experiment_name)


def get_artifact_cache(tag, experiment_name=None):
    assert tag is not None

    if tag.startswith('experiments/'):
        experiment_name = re.sub(
            '\Aexperiments/',
            '',
            re.sub(
                '/[^/]*\Z',
                '',
                tag))
        tag = re.sub('\.tar\.?[^\.]*\Z', '', re.sub('.*/', '', tag))

    if tag.startswith('blobstore/'):
        return get_blob_cache(tag)

    experiment_name = experiment_name if experiment_name else \
        get_experiment_key()
{'fraction_non_alphanumeric': 0.11357603337125521, 'fraction_numerical': 0.00018960940462646946, 'mean_word_length': 2.5665990534144694, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6160586', 'n_tokens_mistral': 1770, 'n_tokens_neox': 1742, 'n_words': 352}