<commit_before>from ..sharding import ShardingSpecification, ShardReader
from ....skeleton import Skeleton
from ..spatial_index import CachedSpatialIndex

class ShardedPrecomputedSkeletonSource(object):
  def __init__(self, meta, cache, config, readonly=False):
    self.meta = meta
    self.cache = cache
    self.config = config
    self.readonly = bool(readonly)

    spec = ShardingSpecification.from_dict(self.meta.info['sharding'])
    self.reader = ShardReader(meta, cache, spec)

    self.spatial_index = None
    if self.meta.spatial_index:
      self.spatial_index = CachedSpatialIndex(
        self.cache,
        cloudpath=self.meta.layerpath, 
        bounds=self.meta.meta.bounds(0) * self.meta.meta.resolution(0),
        chunk_size=self.meta.info['spatial_index']['chunk_size'],
      )

  @property
  def path(self):
    return self.meta.path 

  def get(self, segids):
    list_return = True
    if type(segids) in (int, float):
      list_return = False
      segids = [ int(segids) ]

    # compress = self.config.compress 
    # if compress is None:
    #   compress = True

    results = []
    for segid in segids:
      binary = self.reader.get_data(segid, self.meta.skeleton_path)
      skeleton = Skeleton.from_precomputed(
        binary, segid=segid, 
        vertex_attributes=self.meta.info['vertex_attributes']
      )
      skeleton.transform = self.meta.transform
      results.append(skeleton.physical_space())

    if list_return:
      return results
    else:
      return results[0]

  def upload(self):
    raise NotImplementedError()

  def raw_upload(self):
    raise NotImplementedError()<commit_msg>fix: unintelligible error if using sharded skeleton upload<commit_after>from ..sharding import ShardingSpecification, ShardReader
from ....skeleton import Skeleton
from ..spatial_index import CachedSpatialIndex

class ShardedPrecomputedSkeletonSource(object):
  def __init__(self, meta, cache, config, readonly=False):
    self.meta = meta
    self.cache = cache
    self.config = config
    self.readonly = bool(readonly)

    spec = ShardingSpecification.from_dict(self.meta.info['sharding'])
    self.reader = ShardReader(meta, cache, spec)

    self.spatial_index = None
    if self.meta.spatial_index:
      self.spatial_index = CachedSpatialIndex(
        self.cache,
        cloudpath=self.meta.layerpath, 
        bounds=self.meta.meta.bounds(0) * self.meta.meta.resolution(0),
        chunk_size=self.meta.info['spatial_index']['chunk_size'],
      )

  @property
  def path(self):
    return self.meta.path 

  def get(self, segids):
    list_return = True
    if type(segids) in (int, float):
      list_return = False
      segids = [ int(segids) ]

    # compress = self.config.compress 
    # if compress is None:
    #   compress = True

    results = []
    for segid in segids:
      binary = self.reader.get_data(segid, self.meta.skeleton_path)
      skeleton = Skeleton.from_precomputed(
        binary, segid=segid, 
        vertex_attributes=self.meta.info['vertex_attributes']
      )
      skeleton.transform = self.meta.transform
      results.append(skeleton.physical_space())

    if list_return:
      return results
    else:
      return results[0]

  def upload(self, *args, **kwargs):
    raise NotImplementedError()

  def raw_upload(self, *args, **kwargs):
    raise NotImplementedError()
{'fraction_non_alphanumeric': 0.10437109723461195, 'fraction_numerical': 0.001784121320249777, 'mean_word_length': 3.1633663366336635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3517596', 'n_tokens_mistral': 1084, 'n_tokens_neox': 1029, 'n_words': 218}