<commit_before>import os
import re

from cloudvolume import paths
from cloudvolume.paths import strict_extract, extract, ExtractedPath

from cloudvolume.exceptions import UnsupportedProtocolError
from cloudvolume import lib

def test_path_extraction():
  extract = paths.extract(r'file://C:\wow\this\is\a\cool\path', windows=True, disable_toabs=True)
  print(extract)
  assert extract.protocol == 'file'
  assert extract.bucket == 'C:\\wow\\'

  # on linux the toabs prepends the current path because it
  # doesn't understand C:\... so we can't test that here.
  # assert extract.path == 'this\\is\\a\\cool\\path' 

  try:
    extract = paths.strict_extract(r'file://C:\wow\this\is\a\cool\path', windows=False, disable_toabs=True)
    assert False 
  except UnsupportedProtocolError:
    pass

  def shoulderror(url):
    try:
        paths.strict_extract(url)
        assert False, url
    except:
        pass

  def okgoogle(url):
    path = paths.strict_extract(url)
    assert path.protocol == 'gs', url
    assert path.bucket == 'bucket', url
    assert path.intermediate_path == '', url
    assert path.dataset == 'dataset', url
    assert path.layer == 'layer', url

  okgoogle('gs://bucket/dataset/layer') 
  shoulderror('s4://dataset/layer')
  shoulderror('dataset/layer')
  shoulderror('s3://dataset')

  firstdir = lambda x: '/' + x.split('/')[1]

  homepath = lib.toabs('~')
  homerintermediate = homepath.replace(firstdir(homepath), '')[1:]

  curpath = lib.toabs('.')
  curintermediate = curpath.replace(firstdir(curpath), '')[1:]

  print(curintermediate)

  assert (paths.extract('s3://seunglab-test/intermediate/path/dataset/layer') 
      == ExtractedPath(
        'precomputed', 's3', 'seunglab-test', 
        'intermediate/path/dataset/layer', 'intermediate/path/', 
        'dataset', 'layer'
      ))

  assert (paths.extract('file:///tmp/dataset/layer') 
      == ExtractedPath('precomputed', 'file', "/tmp", 'dataset/layer', '', 'dataset', 'layer'))

  assert (paths.extract('file://seunglab-test/intermediate/path/dataset/layer') 
      == ExtractedPath(
        'precomputed', 'file', '/usr', 
        os.path.join(curintermediate, 'seunglab-test/intermediate/path/dataset/layer'),   
        os.path.join(curintermediate, 'seunglab-test', 'intermediate/path/'), 
       'dataset', 'layer'))

  assert (paths.extract('gs://seunglab-test/intermediate/path/dataset/layer') 
      == ExtractedPath(
        'precomputed', 'gs', 
        'seunglab-test', 'intermediate/path/dataset/layer', 
        'intermediate/path/', 'dataset', 'layer'
      ))

  assert (paths.extract('file://~/seunglab-test/intermediate/path/dataset/layer') 
      == ExtractedPath(
        'precomputed', 'file', <commit_msg>test+fix: use dynamic first directory in test<commit_after>import os
import re

from cloudvolume import paths
from cloudvolume.paths import strict_extract, extract, ExtractedPath

from cloudvolume.exceptions import UnsupportedProtocolError
from cloudvolume import lib

def test_path_extraction():
  extract = paths.extract(r'file://C:\wow\this\is\a\cool\path', windows=True, disable_toabs=True)
  print(extract)
  assert extract.protocol == 'file'
  assert extract.bucket == 'C:\\wow\\'

  # on linux the toabs prepends the current path because it
  # doesn't understand C:\... so we can't test that here.
  # assert extract.path == 'this\\is\\a\\cool\\path' 

  try:
    extract = paths.strict_extract(r'file://C:\wow\this\is\a\cool\path', windows=False, disable_toabs=True)
    assert False 
  except UnsupportedProtocolError:
    pass

  def shoulderror(url):
    try:
        paths.strict_extract(url)
        assert False, url
    except:
        pass

  def okgoogle(url):
    path = paths.strict_extract(url)
    assert path.protocol == 'gs', url
    assert path.bucket == 'bucket', url
    assert path.intermediate_path == '', url
    assert path.dataset == 'dataset', url
    assert path.layer == 'layer', url

  okgoogle('gs://bucket/dataset/layer') 
  shoulderror('s4://dataset/layer')
  shoulderror('dataset/layer')
  shoulderror('s3://dataset')

  firstdir = lambda x: '/' + x.split('/')[1]

  homepath = lib.toabs('~')
  homerintermediate = homepath.replace(firstdir(homepath), '')[1:]

  curpath = lib.toabs('.')
  curintermediate = curpath.replace(firstdir(curpath), '')[1:]

  print(curintermediate)

  assert (paths.extract('s3://seunglab-test/intermediate/path/dataset/layer') 
      == ExtractedPath(
        'precomputed', 's3', 'seunglab-test', 
        'intermediate/path/dataset/layer', 'intermediate/path/', 
        'dataset', 'layer'
      ))

  assert (paths.extract('file:///tmp/dataset/layer') 
      == ExtractedPath('precomputed', 'file', "/tmp", 'dataset/layer', '', 'dataset', 'layer'))

  assert (paths.extract('file://seunglab-test/intermediate/path/dataset/layer') 
      == ExtractedPath(
        'precomputed', 'file', firstdir(curpath), 
        os.path.join(curintermediate, 'seunglab-test/intermediate/path/dataset/layer'),   
        os.path.join(curintermediate, 'seunglab-test', 'intermediate/path/'), 
       'dataset', 'layer'))

  assert (paths.extract('gs://seunglab-test/intermediate/path/dataset/layer') 
      == ExtractedPath(
        'precomputed', 'gs', 
        'seunglab-test', 'intermediate/path/dataset/layer', 
        'intermediate/path/', 'dataset', 'layer'
      ))

  assert (paths.extract('file://~/seunglab-test/intermediate/path/dataset/layer') 
      == ExtractedPath(
        'precomputed', 'file', 
{'fraction_non_alphanumeric': 0.1592936464591298, 'fraction_numerical': 0.0025486983433460767, 'mean_word_length': 3.495908346972177, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5243183', 'n_tokens_mistral': 1800, 'n_tokens_neox': 1717, 'n_words': 334}