<commit_before>import doctest
import logging
import os
import sys
import unittest

_log = logging.getLogger(__name__)
_log.debug("Using virtual case-insensitive file system")

if sys.version_info.major >= 3:
  import io
  file = io.IOBase
  unichr = chr
else:
  import dircache  # in addition to os.listdir


# Patch existence function
_exists = os.path.exists
def exists(path, get = False):
  ''' Case-normalized exists.
      path: file system path to check for existence
      get: if True, returns actual case-corrected path instead of boolean value
      returns: boolean (existence) or string for path
  >>> _saveUnlink("_x.X")
  >>> with open("_x.X", "w"):
  ...   pass  # write using context manager
  >>> print(os.path.exists("_x.X"))
  True
  >>> with open("_x.X", "w"):
  ...   pass  # write over same file using context manager
  >>> print(os.path.exists("_x.X"))
  True
  >>> fd = open("_x.X", "w"); fd.write("123"); fd.close()  # write over same file using file handle
  >>> print(os.path.exists("_x.X"))
  True
  >>> fd = open("_X.x", "w"); fd.write("234"); fd.close()  # write over different case of file using file handle
  >>> print(os.path.exists("_X.x"))
  True
  >>> with open("_X.X", "a") as fd:
  ...   l = fd.write("567")
  >>> fd = open("_x.x", "r"); contents = fd.read(); fd.close(); print(contents)
  234567
  >>> _saveUnlink("_x.X")
  >>> print(os.path.exists("_X.x"))
  False
  >>> print(os.path.exists("_x.X"))
  False
  '''
  _log.debug("Patched os.path.exists %r" % path)
  if type(path) is file: return _exists(path)  # file handle
  if not isinstance(path, (str, bytes, unichr)):
    if not get: return _exists(path)  # delegate or force original Exception
    raise ValueError("Unknown argument type %r" % type(path))
  if path is None or path == "": return _exists(path) if not get else path
  steps = path.split(os.sep)
  if steps[0] == "": steps[0] = os.sep # absolute
  elif steps[0] != os.curdir: steps.insert(0, os.curdir)
  i = 1
  while "" in steps[i:]: i = steps.index("", i); steps.pop(i)
  path2 = ""
  files = {steps[0].upper(): steps[0]}  # initial mapping
  for step in steps:
    if step == os.pardir:
      if os.sep in path2: path2 = path2[:path2.rindex(os.sep)]; continue  # remove intermediate "..""
      else: path2 = os.pardir  # if ".."" is first entry
    try: path2 += (os.sep if step not in (os.sep, os.curdir) and path2 != os.sep else "") + files[step.upper()]
    except KeyError as E:  # file entry not found
      return False if not get else path2 + (os.sep if step not in (os.sep, os.curdir) and path2 != os.sep else "") + step  # file name not in last folder: doesn't exist (yet)
#    if _isfile(path2): return True if not get else path2  # TODO calls os.stat which was already patched
    try: files = {f.upper(): f for f in _listdir(path2)}; continue
    except IOError as E: pass
    except OSError as E: pass
    try:
      with _open(path2, "rb") as fd: return True if not get else path2
    except: False if not get else path2
  return True if not get else path2<commit_msg>Remv. unichar<commit_after>import doctest
import logging
import os
import sys
import unittest

_log = logging.getLogger(__name__)
_log.debug("Using virtual case-insensitive file system")

if sys.version_info.major >= 3:
  import io
  file = io.IOBase
else:
  import dircache  # in addition to os.listdir


# Patch existence function
_exists = os.path.exists
def exists(path, get = False):
  ''' Case-normalized exists.
      path: file system path to check for existence
      get: if True, returns actual case-corrected path instead of boolean value
      returns: boolean (existence) or string for path
  >>> _saveUnlink("_x.X")
  >>> with open("_x.X", "w"):
  ...   pass  # write using context manager
  >>> print(os.path.exists("_x.X"))
  True
  >>> with open("_x.X", "w"):
  ...   pass  # write over same file using context manager
  >>> print(os.path.exists("_x.X"))
  True
  >>> fd = open("_x.X", "w"); fd.write("123"); fd.close()  # write over same file using file handle
  >>> print(os.path.exists("_x.X"))
  True
  >>> fd = open("_X.x", "w"); fd.write("234"); fd.close()  # write over different case of file using file handle
  >>> print(os.path.exists("_X.x"))
  True
  >>> with open("_X.X", "a") as fd:
  ...   l = fd.write("567")
  >>> fd = open("_x.x", "r"); contents = fd.read(); fd.close(); print(contents)
  234567
  >>> _saveUnlink("_x.X")
  >>> print(os.path.exists("_X.x"))
  False
  >>> print(os.path.exists("_x.X"))
  False
  '''
  _log.debug("Patched os.path.exists %r" % path)
  if type(path) is file: return _exists(path)  # file handle
  if not isinstance(path, (str, bytes)):
    if not get: return _exists(path)  # delegate or force original Exception
    raise ValueError("Unknown argument type %r" % type(path))
  if path is None or path == "": return _exists(path) if not get else path
  steps = path.split(os.sep)
  if steps[0] == "": steps[0] = os.sep # absolute
  elif steps[0] != os.curdir: steps.insert(0, os.curdir)
  i = 1
  while "" in steps[i:]: i = steps.index("", i); steps.pop(i)
  path2 = ""
  files = {steps[0].upper(): steps[0]}  # initial mapping
  for step in steps:
    if step == os.pardir:
      if os.sep in path2: path2 = path2[:path2.rindex(os.sep)]; continue  # remove intermediate "..""
      else: path2 = os.pardir  # if ".."" is first entry
    try: path2 += (os.sep if step not in (os.sep, os.curdir) and path2 != os.sep else "") + files[step.upper()]
    except KeyError as E:  # file entry not found
      return False if not get else path2 + (os.sep if step not in (os.sep, os.curdir) and path2 != os.sep else "") + step  # file name not in last folder: doesn't exist (yet)
#    if _isfile(path2): return True if not get else path2  # TODO calls os.stat which was already patched
    try: files = {f.upper(): f for f in _listdir(path2)}; continue
    except IOError as E: pass
    except OSError as E: pass
    try:
      with _open(path2, "rb") as fd: return True if not get else path2
    except: False if not get else path2
  return True if not get else path2
{'fraction_non_alphanumeric': 0.15264284538119546, 'fraction_numerical': 0.013173061090070805, 'mean_word_length': 3.304748405386251, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4948762', 'n_tokens_mistral': 2103, 'n_tokens_neox': 2062, 'n_words': 703}