<commit_before>"""Common file operations."""

import io
import os
import sys

import controlflow
import display
from var import GM_Globals
from var import GM_SYS_ENCODING
from var import UTF8_SIG


def _open_file(filename, mode, encoding=None, newline=None):
  """Opens a file with no error handling."""
  # Determine which encoding to use
  if 'b' in mode:
    encoding = None
  elif not encoding:
    encoding = GM_Globals[GM_SYS_ENCODING]
  elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
    encoding = UTF8_SIG

  return open(
      os.path.expanduser(filename), mode, newline=newline, encoding=encoding)


def open_file(filename,
              mode='r',
              encoding=None,
              newline=None,
              strip_utf_bom=False):
  """Opens a file.

  Args:
    filename: String, the name of the file to open, or '-' to use stdin/stdout,
      to read/write, depending on the mode param, respectively.
    mode: String, the common file mode to open the file with. Default is read.
    encoding: String, the name of the encoding used to decode or encode the
      file. This should only be used in text mode.
    newline: See param description in
        https://docs.python.org/3.7/library/functions.html#open
    strip_utf_bom: Boolean, True if the file being opened should seek past the
      UTF Byte Order Mark before being returned.
        See more: https://en.wikipedia.org/wiki/UTF-8#Byte_order_mark

  Returns:
    The opened file.
  """
  try:
    if filename == '-':
      # Read from stdin, rather than a file
      if 'r' in mode:
        return io.StringIO(str(sys.stdin.read()))
      return sys.stdout

    # Open a file on disk
    f = _open_file(filename, mode, newline=newline, encoding=encoding)
    if strip_utf_bom:
      utf_bom = u'\ufeff'
      has_bom = False

      if 'b' in mode:
        has_bom = f.read(3).decode('UTF-8') == utf_bom
      elif f.encoding and not f.encoding.lower().startswith('utf'):
        # Convert UTF BOM into ISO-8859-1 via Bytes
        utf8_bom_bytes = utf_bom.encode('UTF-8')
        iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
            'iso-8859-1')
        has_bom = f.read(3).encode('iso-8859-1', 'replace') == iso_8859_1_bom
      else:
        has_bom = f.read(1) == utf_bom

      if not has_bom:
        f.seek(0)

    return f

  except IOError as e:
    controlflow.system_error_exit(6, e)


def close_file(f, force_flush=False):
  """Closes a file.

  Args:
    f: The file to close
    force_flush: Flush file to disk emptying Python and OS caches. See:
       https://stackoverflow.com/a/13762137/1503886

  Returns:
    Boolean, True if the file was successfully closed. False if an error
        was encountered while closing.
  """
  if force_to_disk:
    f.flush()
    os.fsync(f.fileno())
  try:
    f.close()
    return True
  except IOError as e:
    display.print_error(e)
    return False


def read_file(filename,
              mode='r',
              encoding=None,
              newline=None,
              continue_on_error=False,
              display_errors=True):
  """Reads a file from disk.<commit_msg>fix force variable name<commit_after>"""Common file operations."""

import io
import os
import sys

import controlflow
import display
from var import GM_Globals
from var import GM_SYS_ENCODING
from var import UTF8_SIG


def _open_file(filename, mode, encoding=None, newline=None):
  """Opens a file with no error handling."""
  # Determine which encoding to use
  if 'b' in mode:
    encoding = None
  elif not encoding:
    encoding = GM_Globals[GM_SYS_ENCODING]
  elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
    encoding = UTF8_SIG

  return open(
      os.path.expanduser(filename), mode, newline=newline, encoding=encoding)


def open_file(filename,
              mode='r',
              encoding=None,
              newline=None,
              strip_utf_bom=False):
  """Opens a file.

  Args:
    filename: String, the name of the file to open, or '-' to use stdin/stdout,
      to read/write, depending on the mode param, respectively.
    mode: String, the common file mode to open the file with. Default is read.
    encoding: String, the name of the encoding used to decode or encode the
      file. This should only be used in text mode.
    newline: See param description in
        https://docs.python.org/3.7/library/functions.html#open
    strip_utf_bom: Boolean, True if the file being opened should seek past the
      UTF Byte Order Mark before being returned.
        See more: https://en.wikipedia.org/wiki/UTF-8#Byte_order_mark

  Returns:
    The opened file.
  """
  try:
    if filename == '-':
      # Read from stdin, rather than a file
      if 'r' in mode:
        return io.StringIO(str(sys.stdin.read()))
      return sys.stdout

    # Open a file on disk
    f = _open_file(filename, mode, newline=newline, encoding=encoding)
    if strip_utf_bom:
      utf_bom = u'\ufeff'
      has_bom = False

      if 'b' in mode:
        has_bom = f.read(3).decode('UTF-8') == utf_bom
      elif f.encoding and not f.encoding.lower().startswith('utf'):
        # Convert UTF BOM into ISO-8859-1 via Bytes
        utf8_bom_bytes = utf_bom.encode('UTF-8')
        iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
            'iso-8859-1')
        has_bom = f.read(3).encode('iso-8859-1', 'replace') == iso_8859_1_bom
      else:
        has_bom = f.read(1) == utf_bom

      if not has_bom:
        f.seek(0)

    return f

  except IOError as e:
    controlflow.system_error_exit(6, e)


def close_file(f, force_flush=False):
  """Closes a file.

  Args:
    f: The file to close
    force_flush: Flush file to disk emptying Python and OS caches. See:
       https://stackoverflow.com/a/13762137/1503886

  Returns:
    Boolean, True if the file was successfully closed. False if an error
        was encountered while closing.
  """
  if force_flush:
    f.flush()
    os.fsync(f.fileno())
  try:
    f.close()
    return True
  except IOError as e:
    display.print_error(e)
    return False


def read_file(filename,
              mode='r',
              encoding=None,
              newline=None,
              continue_on_error=False,
              display_errors=True):
  """Reads a file from disk.
{'fraction_non_alphanumeric': 0.10082435003170577, 'fraction_numerical': 0.019023462270133164, 'mean_word_length': 2.5603837471783297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '182973', 'n_tokens_mistral': 2103, 'n_tokens_neox': 2000, 'n_words': 628}