<commit_before>import six

from collections import defaultdict
import itertools
import json
import os
import posixpath
import re
import requests

import numpy as np
from tqdm import tqdm

from ...lib import red, toiter, Bbox
from ...mesh import Mesh
from ... import paths
from ...storage import Storage, GreenStorage

from ..precomputed.mesh import PrecomputedMeshSource

class GrapheneMeshSource(PrecomputedMeshSource):
  def _get_fragment_filenames(self, seg_id, lod=0, level=2, bbox=None):
    # TODO: add lod to endpoint
    query_d = {
      'verify': True,
    }

    if bbox is not None:
      bbox = Bbox.create(bbox)
      query_d['bounds'] = bbox.to_filename()

    url = "%s/%s:%s" % (self.meta.manifest_endpoint, seg_id, lod)
    
    if level is not None:
      res = requests.get(
        url,
        data=json.dumps({ "start_layer": level }),
        params=query_d,
        headers=self.meta.auth_header
      )
    else:
      res = requests.get(url, params=query_d, headers=self.meta.auth_header)

    res.raise_for_status()

    return json.loads(res.content)["fragments"]

  def get(
      self, segids, 
      remove_duplicate_vertices=False, 
      fuse=False, level=2, 
      bounding_box=None
    ):
    """
    Merge fragments derived from these segids into a single vertex and face list.

    Why merge multiple segids into one mesh? For example, if you have a set of
    segids that belong to the same neuron.

    segid: (iterable or int) segids to render into a single mesh

    Optional:<commit_msg>fix: decode fragment filenames to utf8 before json parsing<commit_after>import six

from collections import defaultdict
import itertools
import json
import os
import posixpath
import re
import requests

import numpy as np
from tqdm import tqdm

from ...lib import red, toiter, Bbox
from ...mesh import Mesh
from ... import paths
from ...storage import Storage, GreenStorage

from ..precomputed.mesh import PrecomputedMeshSource

class GrapheneMeshSource(PrecomputedMeshSource):
  def _get_fragment_filenames(self, seg_id, lod=0, level=2, bbox=None):
    # TODO: add lod to endpoint
    query_d = {
      'verify': True,
    }

    if bbox is not None:
      bbox = Bbox.create(bbox)
      query_d['bounds'] = bbox.to_filename()

    url = "%s/%s:%s" % (self.meta.manifest_endpoint, seg_id, lod)
    
    if level is not None:
      res = requests.get(
        url,
        data=json.dumps({ "start_layer": level }),
        params=query_d,
        headers=self.meta.auth_header
      )
    else:
      res = requests.get(url, params=query_d, headers=self.meta.auth_header)

    res.raise_for_status()

    return json.loads(res.content.decode('utf8'))["fragments"]

  def get(
      self, segids, 
      remove_duplicate_vertices=False, 
      fuse=False, level=2, 
      bounding_box=None
    ):
    """
    Merge fragments derived from these segids into a single vertex and face list.

    Why merge multiple segids into one mesh? For example, if you have a set of
    segids that belong to the same neuron.

    segid: (iterable or int) segids to render into a single mesh

    Optional:
{'fraction_non_alphanumeric': 0.08974358974358974, 'fraction_numerical': 0.002564102564102564, 'mean_word_length': 3.0427461139896375, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3210361', 'n_tokens_mistral': 997, 'n_tokens_neox': 973, 'n_words': 315}