<commit_before># Simplified from peyotl.phylesystem.phylesystem_umbrella
# TODO:
#   - clobber all references to study prefix (eg, 'pg')
#   - add uniqueness test for collection_id instead
#   - a collection id should be a unique "path" string composed 
#     of '{ownerid}/{slugified-collection-name}'
#     EXAMPLES: 'jimallman/trees-about-bees'
#               'jimallman/other-interesting-stuff'
#               'kcranston/trees-about-bees'
#               'jimallman/trees-about-bees-2'
#   - refactor shard to base class, with generalized 'items'?
from peyotl.utility import get_logger
from peyotl.utility.str_util import slugify, \
                                    increment_slug
try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO
import json
try:
    import anyjson
except:
    class Wrapper(object):
        pass
    anyjson = Wrapper()
    anyjson.loads = json.loads
try:
    from dogpile.cache.api import NO_VALUE
except:
    pass #caching is optional
from peyotl.git_storage import ShardedDocStore, \
                               TypeAwareDocStore
from peyotl.collections.collections_shard import TreeCollectionsShardProxy, \
                                                 TreeCollectionsShard
from peyotl.collections import get_empty_collection
from peyotl.collection_validation import validate_collection
from peyotl.collections.git_actions import TreeCollectionsGitAction
#from peyotl.phylesystem.git_workflows import commit_and_try_merge2master, \
#                                             delete_study, \
#                                             validate_and_convert_nexson
#from peyotl.nexson_validation import ot_validate
from threading import Lock
import os
import re

OWNER_ID_PATTERN =      re.compile(r'^[a-zA-Z0-9-]+$')
COLLECTION_ID_PATTERN = re.compile(r'^[a-zA-Z0-9-]+/[a-z0-9-]+$')
# Allow simple slug-ified strings and slash separator (no whitespace!)

_LOG = get_logger(__name__)

def prefix_from_collection_path(collection_id):
    # The collection id is a sort of "path", e.g. '{ownerid}/{collection-name-as-slug}'
    #   EXAMPLES: 'jimallman/trees-about-bees', 'kcranston/interesting-trees-2'
    # Assume that the ownerid will work as a prefix, esp. by assigning all of a
    # user's collections to a single shard.for grouping in shards
    path_parts = collection_id.split('/')
    if len(path_parts) > 1:
        return path_parts[0]
    return 'anonymous'  # or perhaps None?

class TreeCollectionStoreProxy(ShardedDocStore):
    '''Proxy for interacting with external resources if given the configuration of a remote TreeCollectionStore
    '''
    def __init__(self, config):
        ShardedDocStore.__init__(self,
                                 prefix_from_doc_id=prefix_from_collection_path)
        for s in config.get('shards', []):
            self._shards.append(TreeCollectionsShardProxy(s))
        d = {}
        for s in self._shards:
            for k in s.doc_index.keys():<commit_msg>Diagnostic junk for failed prefix-test<commit_after># Simplified from peyotl.phylesystem.phylesystem_umbrella
# TODO:
#   - clobber all references to study prefix (eg, 'pg')
#   - add uniqueness test for collection_id instead
#   - a collection id should be a unique "path" string composed 
#     of '{ownerid}/{slugified-collection-name}'
#     EXAMPLES: 'jimallman/trees-about-bees'
#               'jimallman/other-interesting-stuff'
#               'kcranston/trees-about-bees'
#               'jimallman/trees-about-bees-2'
#   - refactor shard to base class, with generalized 'items'?
from peyotl.utility import get_logger
from peyotl.utility.str_util import slugify, \
                                    increment_slug
try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO
import json
try:
    import anyjson
except:
    class Wrapper(object):
        pass
    anyjson = Wrapper()
    anyjson.loads = json.loads
try:
    from dogpile.cache.api import NO_VALUE
except:
    pass #caching is optional
from peyotl.git_storage import ShardedDocStore, \
                               TypeAwareDocStore
from peyotl.collections.collections_shard import TreeCollectionsShardProxy, \
                                                 TreeCollectionsShard
from peyotl.collections import get_empty_collection
from peyotl.collection_validation import validate_collection
from peyotl.collections.git_actions import TreeCollectionsGitAction
#from peyotl.phylesystem.git_workflows import commit_and_try_merge2master, \
#                                             delete_study, \
#                                             validate_and_convert_nexson
#from peyotl.nexson_validation import ot_validate
from threading import Lock
import os
import re

OWNER_ID_PATTERN =      re.compile(r'^[a-zA-Z0-9-]+$')
COLLECTION_ID_PATTERN = re.compile(r'^[a-zA-Z0-9-]+/[a-z0-9-]+$')
# Allow simple slug-ified strings and slash separator (no whitespace!)

_LOG = get_logger(__name__)

def prefix_from_collection_path(collection_id):
    # The collection id is a sort of "path", e.g. '{ownerid}/{collection-name-as-slug}'
    #   EXAMPLES: 'jimallman/trees-about-bees', 'kcranston/interesting-trees-2'
    # Assume that the ownerid will work as a prefix, esp. by assigning all of a
    # user's collections to a single shard.for grouping in shards
    _LOG.debug('> prefix_from_collection_path(), testing this id: {i}'.format(i=collection_id))
    path_parts = collection_id.split('/')
    _LOG.debug('> prefix_from_collection_path(), found {} path parts'.format(len(path_parts)))
    if len(path_parts) > 1:
        return path_parts[0]
    return 'anonymous'  # or perhaps None?

class TreeCollectionStoreProxy(ShardedDocStore):
    '''Proxy for interacting with external resources if given the configuration of a remote TreeCollectionStore
    '''
    def __init__(self, config):
        ShardedDocStore.__init__(self,
                                 prefix_from_doc_id=prefix_from_collection_path)
        for s in config.get('shards', []):
            self._shards.append(TreeCollectionsShardProxy(s))
        d = {}
        for s in self._shards:
            for k in s.doc_index.keys():
{'fraction_non_alphanumeric': 0.08626249596904224, 'fraction_numerical': 0.003547242824895195, 'mean_word_length': 2.81019656019656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7138875', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1838, 'n_words': 470}