<commit_before># BEGIN_COPYRIGHT
# 
# Copyright 2009-2014 CRS4.
# 
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# 
# END_COPYRIGHT

# pylint: disable=W0311

import os, unittest, tempfile
from numbers import Number

import pydoop.hdfs as hdfs
from pydoop.hdfs.common import DEFAULT_PORT, DEFAULT_USER
from pydoop.utils import make_random_str

from utils import UNI_CHR


def uni_last(tup):
  return tup[:-1] + (tup[-1]+UNI_CHR,)


class TestSplit(unittest.TestCase):

  def good(self):
    cases = [
      ('hdfs://localhost:9000/', ('localhost', 9000, '/')),
      ('hdfs://localhost:9000/a/b', ('localhost', 9000, '/a/b')),
      ('hdfs://localhost/a/b', ('localhost', DEFAULT_PORT, '/a/b')),
      ('hdfs:///a/b', ('default', 0, '/a/b')),
      ('hdfs:/', ('default', 0, '/')),
      ('file:///a/b', ('', 0, '/a/b')),
      ('file:/a/b', ('', 0, '/a/b')),
      ('file:///a', ('', 0, '/a')),
      ('file:/a', ('', 0, '/a')),
      ('file://temp/foo.txt', ('', 0, 'temp/foo.txt')),
      ('file://temp', ('', 0, 'temp')),
      ]
    if hdfs.default_is_local():
      cases.extend([
        ('///a/b', ('', 0, '/a/b')),
        ('/a/b', ('', 0, '/a/b')),
        ('a/b', ('', 0, 'a/b')),
        ])
    else:
      cases.extend([
        ('///a/b', ('default', 0, '/a/b')),
        ('/a/b', ('default', 0, '/a/b')),
        ('a/b', ('default', 0, '/user/%s/a/b' % DEFAULT_USER)),
        ])
    for p, r in cases:
      self.assertEqual(hdfs.path.split(p), r)
    for p, r in cases[1:]:
      self.assertEqual(hdfs.path.split(p+UNI_CHR), uni_last(r))

  def good_with_user(self):
    if hdfs.default_is_local():
      cases = [('a/b', u, ('', 0, 'a/b')) for u in None, DEFAULT_USER, 'foo']
    else:
      cases = [
        ('a/b', None, ('default', 0, '/user/%s/a/b' % DEFAULT_USER)),
        ('a/b', DEFAULT_USER, ('default', 0, '/user/%s/a/b' % DEFAULT_USER)),
        ('a/b', 'foo', ('default', 0, '/user/foo/a/b')),
        ]
    for p, u, r in cases:
      self.assertEqual(hdfs.path.split(p, u), r)
      self.assertEqual(hdfs.path.split(p+UNI_CHR, u), uni_last(r))

  def bad(self):
    cases = [
      '',                                  # not allowed in the Java API
      'hdfs:',                             # no scheme-specific part
      'hdfs://',                           # path part is empty
      'ftp://localhost:9000/',             # bad scheme
      'hdfs://localhost:spam/',            # port is not an int
      'hdfs://localhost:9000',             # path part is empty
      'hdfs://localhost:9000/a:b',         # colon outside netloc
      '//localhost:9000/a/b',              # null scheme
      ]
    if not hdfs.default_is_local():
      cases.append('/localhost:9000/a/b')  # colon outside netloc
    for p in cases:
      self.assertRaises(ValueError, hdfs.path.split, p)

  def splitext(self):
    for pre in '', 'file:', 'hdfs://host:1':
      name, ext = '%sfoo' % pre, '.txt'
      self.assertEqual(hdfs.path.splitext(name+ext), (name, ext))


class TestUnparse(unittest.TestCase):

  def good(self):
    cases = [<commit_msg>added a test for splitext<commit_after># BEGIN_COPYRIGHT
# 
# Copyright 2009-2014 CRS4.
# 
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# 
# END_COPYRIGHT

# pylint: disable=W0311

import os, unittest, tempfile
from numbers import Number

import pydoop.hdfs as hdfs
from pydoop.hdfs.common import DEFAULT_PORT, DEFAULT_USER
from pydoop.utils import make_random_str

from utils import UNI_CHR


def uni_last(tup):
  return tup[:-1] + (tup[-1]+UNI_CHR,)


class TestSplit(unittest.TestCase):

  def good(self):
    cases = [
      ('hdfs://localhost:9000/', ('localhost', 9000, '/')),
      ('hdfs://localhost:9000/a/b', ('localhost', 9000, '/a/b')),
      ('hdfs://localhost/a/b', ('localhost', DEFAULT_PORT, '/a/b')),
      ('hdfs:///a/b', ('default', 0, '/a/b')),
      ('hdfs:/', ('default', 0, '/')),
      ('file:///a/b', ('', 0, '/a/b')),
      ('file:/a/b', ('', 0, '/a/b')),
      ('file:///a', ('', 0, '/a')),
      ('file:/a', ('', 0, '/a')),
      ('file://temp/foo.txt', ('', 0, 'temp/foo.txt')),
      ('file://temp', ('', 0, 'temp')),
      ]
    if hdfs.default_is_local():
      cases.extend([
        ('///a/b', ('', 0, '/a/b')),
        ('/a/b', ('', 0, '/a/b')),
        ('a/b', ('', 0, 'a/b')),
        ])
    else:
      cases.extend([
        ('///a/b', ('default', 0, '/a/b')),
        ('/a/b', ('default', 0, '/a/b')),
        ('a/b', ('default', 0, '/user/%s/a/b' % DEFAULT_USER)),
        ])
    for p, r in cases:
      self.assertEqual(hdfs.path.split(p), r)
    for p, r in cases[1:]:
      self.assertEqual(hdfs.path.split(p+UNI_CHR), uni_last(r))

  def good_with_user(self):
    if hdfs.default_is_local():
      cases = [('a/b', u, ('', 0, 'a/b')) for u in None, DEFAULT_USER, 'foo']
    else:
      cases = [
        ('a/b', None, ('default', 0, '/user/%s/a/b' % DEFAULT_USER)),
        ('a/b', DEFAULT_USER, ('default', 0, '/user/%s/a/b' % DEFAULT_USER)),
        ('a/b', 'foo', ('default', 0, '/user/foo/a/b')),
        ]
    for p, u, r in cases:
      self.assertEqual(hdfs.path.split(p, u), r)
      self.assertEqual(hdfs.path.split(p+UNI_CHR, u), uni_last(r))

  def bad(self):
    cases = [
      '',                                  # not allowed in the Java API
      'hdfs:',                             # no scheme-specific part
      'hdfs://',                           # path part is empty
      'ftp://localhost:9000/',             # bad scheme
      'hdfs://localhost:spam/',            # port is not an int
      'hdfs://localhost:9000',             # path part is empty
      'hdfs://localhost:9000/a:b',         # colon outside netloc
      '//localhost:9000/a/b',              # null scheme
      ]
    if not hdfs.default_is_local():
      cases.append('/localhost:9000/a/b')  # colon outside netloc
    for p in cases:
      self.assertRaises(ValueError, hdfs.path.split, p)

  def splitext(self):
    for pre in '', 'file:', 'hdfs://host:1':
      name, ext = '%sfoo' % pre, '.txt'
      self.assertEqual(hdfs.path.splitext(name+ext), (name, ext))
    p = 'hdfs://foo.com:1/'
    self.assertEqual(hdfs.path.splitext(p), (p, ''))


class TestUnparse(unittest.TestCase):

  def good(self):
    cases = [
{'fraction_non_alphanumeric': 0.20030475135060258, 'fraction_numerical': 0.02091702451863139, 'mean_word_length': 2.358139534883721, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6914383', 'n_tokens_mistral': 2685, 'n_tokens_neox': 2547, 'n_words': 642}