<commit_before>#!/usr/bin/env python
'''Utilities for dealing with local filesystem
copies of the phylesystem repositories.
'''
from peyotl.utility import get_config, expand_path, get_logger
from cStringIO import StringIO
import json
try:
    import anyjson
except:
    class Wrapper(object):
        pass
    anyjson = Wrapper()
    anyjson.loads = json.loads
try:
    from dogpile.cache.api import NO_VALUE
except:
    pass #caching is optional
from peyotl.phylesystem.git_actions import GitAction, \
                                           get_filepath_for_namespaced_id, \
                                           get_filepath_for_simple_id
from peyotl.phylesystem.git_workflows import commit_and_try_merge2master, \
                                             delete_study, \
                                             validate_and_convert_nexson
from peyotl.nexson_syntax import detect_nexson_version
from peyotl.nexson_validation import ot_validate
from peyotl.nexson_validation._validation_base import NexsonAnnotationAdder, \
                                                      replace_same_agent_annotation
import codecs
import os
import re
from threading import Lock
_LOG = get_logger(__name__)
_study_index_lock = Lock()
_study_index = None

STUDY_ID_PATTERN = re.compile(r'^[a-zA-Z]+_+[0-9]+$')
def _get_phylesystem_parent_with_source():
    src = 'environment'
    if 'PHYLESYSTEM_PARENT' in os.environ:
        phylesystem_parent = os.environ.get('PHYLESYSTEM_PARENT')
    else:
        try:
            phylesystem_parent = expand_path(get_config('phylesystem', 'parent'))
            src = 'configfile'
        except:
            raise ValueError('No phylesystem parent specified in config or environmental variables')
    x = phylesystem_parent.split(':') #TEMP hardcoded assumption that : does not occur in a path name
    return x, src

def _get_phylesystem_parent():
    return _get_phylesystem_parent_with_source()[0]


def get_repos(par_list=None):
    '''Returns a dictionary of name -> filepath
    `name` is the repo name based on the dir name (not the get repo). It is not
        terribly useful, but it is nice to have so that any mirrored repo directory can
        use the same naming convention.
    `filepath` will be the full path to the repo directory (it will end in `name`)
    '''
    _repos = {} # key is repo name, value repo location
    if par_list is None:
        par_list = _get_phylesystem_parent()
    elif not isinstance(par_list, list):
        par_list = [par_list]
    for p in par_list:
        if not os.path.isdir(p):
            raise ValueError('No phylesystem parent "{p}" is not a directory'.format(p=p))
        for name in os.listdir(p):
            if os.path.isdir(os.path.join(p, name + '/.git')):
                _repos[name] = os.path.abspath(os.path.join(p, name))
    if len(_repos) == 0:
        raise ValueError('No git repos in {parent}'.format(parent=str(par_list)))<commit_msg>better exception message<commit_after>#!/usr/bin/env python
'''Utilities for dealing with local filesystem
copies of the phylesystem repositories.
'''
from peyotl.utility import get_config, expand_path, get_logger
from cStringIO import StringIO
import json
try:
    import anyjson
except:
    class Wrapper(object):
        pass
    anyjson = Wrapper()
    anyjson.loads = json.loads
try:
    from dogpile.cache.api import NO_VALUE
except:
    pass #caching is optional
from peyotl.phylesystem.git_actions import GitAction, \
                                           get_filepath_for_namespaced_id, \
                                           get_filepath_for_simple_id
from peyotl.phylesystem.git_workflows import commit_and_try_merge2master, \
                                             delete_study, \
                                             validate_and_convert_nexson
from peyotl.nexson_syntax import detect_nexson_version
from peyotl.nexson_validation import ot_validate
from peyotl.nexson_validation._validation_base import NexsonAnnotationAdder, \
                                                      replace_same_agent_annotation
import codecs
import os
import re
from threading import Lock
_LOG = get_logger(__name__)
_study_index_lock = Lock()
_study_index = None

STUDY_ID_PATTERN = re.compile(r'^[a-zA-Z]+_+[0-9]+$')
def _get_phylesystem_parent_with_source():
    src = 'environment'
    if 'PHYLESYSTEM_PARENT' in os.environ:
        phylesystem_parent = os.environ.get('PHYLESYSTEM_PARENT')
    else:
        try:
            phylesystem_parent = expand_path(get_config('phylesystem', 'parent'))
            src = 'configfile'
        except:
            raise ValueError('No [phylesystem] "parent" specified in config or environmental variables')
    x = phylesystem_parent.split(':') #TEMP hardcoded assumption that : does not occur in a path name
    return x, src

def _get_phylesystem_parent():
    return _get_phylesystem_parent_with_source()[0]


def get_repos(par_list=None):
    '''Returns a dictionary of name -> filepath
    `name` is the repo name based on the dir name (not the get repo). It is not
        terribly useful, but it is nice to have so that any mirrored repo directory can
        use the same naming convention.
    `filepath` will be the full path to the repo directory (it will end in `name`)
    '''
    _repos = {} # key is repo name, value repo location
    if par_list is None:
        par_list = _get_phylesystem_parent()
    elif not isinstance(par_list, list):
        par_list = [par_list]
    for p in par_list:
        if not os.path.isdir(p):
            raise ValueError('No phylesystem parent "{p}" is not a directory'.format(p=p))
        for name in os.listdir(p):
            if os.path.isdir(os.path.join(p, name + '/.git')):
                _repos[name] = os.path.abspath(os.path.join(p, name))
    if len(_repos) == 0:
        raise ValueError('No git repos in {parent}'.format(parent=str(par_list)))
{'fraction_non_alphanumeric': 0.08087615838247683, 'fraction_numerical': 0.0016849199663016006, 'mean_word_length': 2.6574245224892175, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1053731', 'n_tokens_mistral': 1760, 'n_tokens_neox': 1726, 'n_words': 507}