<commit_before># Copyright 2018 The Lucid Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Methods for read_handle bytes from arbitrary sources.

This module takes a URL, infers how to locate it,
loads the data into memory and returns it.
"""
from contextlib import contextmanager
import hashlib
import os
import re
import logging
from urllib.parse import urlparse
from urllib import request
from tensorflow.io import gfile
import tensorflow as tf
from tempfile import gettempdir
import gc
from filelock import FileLock

from lucid.misc.io.writing import write_handle
from lucid.misc.io.scoping import scope_url


# create logger with module name, e.g. lucid.misc.io.reading
log = logging.getLogger(__name__)


# Public functions


def read(url, encoding=None, cache=None, mode="rb"):
    """Read from any URL.

    Internally differentiates between URLs supported by tf.gfile, such as URLs
    with the Google Cloud Storage scheme ('gs://...') or local paths, and HTTP
    URLs. This way users don't need to know about the underlying fetch mechanism.

    Args:
        url: a URL including scheme or a local path
        mode: mode in which to open the file. defaults to binary ('rb')
        encoding: if specified, encoding that should be used to decode read data
          if mode is specified to be text ('r'), this defaults to 'utf-8'.
        cache: whether to attempt caching the resource. Defaults to True only if
          the given URL specifies a remote resource.
    Returns:
        All bytes form the specified resource, or a decoded string of those.
    """
    with read_handle(url, cache, mode=mode) as handle:
        data = handle.read()

    if encoding:
        data = data.decode(encoding)

    return data


@contextmanager
def read_handle(url, cache=None, mode="rb"):
    """Read from any URL with a file handle.

    Use this to get a handle to a file rather than eagerly load the data:

    ```
    with read_handle(url) as handle:
    result = something.load(handle)

    result.do_something()

    ```

    When program execution leaves this `with` block, the handle will be closed
    automatically.

    Args:
        url: a URL including scheme or a local path
    Returns:
        A file handle to the specified resource if it could be reached.
        The handle will be closed automatically once execution leaves this context.
    """
    url = scope_url(url)

    scheme = urlparse(url).scheme

    if cache == "purge":
        _purge_cached(url)
        cache = None

    if _is_remote(scheme) and cache is None:
        cache = True
        log.debug("Cache not specified, enabling because resource is remote.")

    if cache:
        handle = _read_and_cache(url, mode=mode)
    else:
        if scheme in ("http", "https"):
            handle = _handle_web_url(url, mode=mode)
        elif scheme in ("gs"):
            handle = _handle_gfile(url, mode=mode)
        else:
            handle = open(url, mode=mode)

    yield handle
    handle.close()


# Handlers


def _handle_gfile(url, mode="rb"):
    return gfile.Open(url, mode)


def _handle_web_url(url, mode="r"):
    return request.urlopen(url)


# Helper Functions


def _is_remote(scheme):
    return scheme in ("http", "https", "gs")


RESERVED_PATH_CHARS = re.compile("[^a-zA-Z0-9]")
LUCID_CACHE_DIR_NAME = "lucid_cache"
MAX_FILENAME_LENGTH = 200
_LUCID_CACHE_DIR = None  # filled on first use


def local_cache_path(remote_url):
    global _LUCID_CACHE_DIR
    """Returns the path that remote_url would be cached at locally."""
    local_name = RESERVED_PATH_CHARS.sub("_", remote_url)
    if len(local_name) > MAX_FILENAME_LENGTH:
        filename_hash = hashlib.sha256(local_name.encode("utf-8")).hexdigest()
        truncated_name = (
            local_name[: (MAX_FILENAME_LENGTH - (len(filename_hash)) - 1)]
            + "-"<commit_msg>Correct way to call modern GFile<commit_after># Copyright 2018 The Lucid Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Methods for read_handle bytes from arbitrary sources.

This module takes a URL, infers how to locate it,
loads the data into memory and returns it.
"""
from contextlib import contextmanager
import hashlib
import os
import re
import logging
from urllib.parse import urlparse
from urllib import request
from tensorflow.io.gfile import GFile
import tensorflow as tf
from tempfile import gettempdir
import gc
from filelock import FileLock

from lucid.misc.io.writing import write_handle
from lucid.misc.io.scoping import scope_url


# create logger with module name, e.g. lucid.misc.io.reading
log = logging.getLogger(__name__)


# Public functions


def read(url, encoding=None, cache=None, mode="rb"):
    """Read from any URL.

    Internally differentiates between URLs supported by tf.gfile, such as URLs
    with the Google Cloud Storage scheme ('gs://...') or local paths, and HTTP
    URLs. This way users don't need to know about the underlying fetch mechanism.

    Args:
        url: a URL including scheme or a local path
        mode: mode in which to open the file. defaults to binary ('rb')
        encoding: if specified, encoding that should be used to decode read data
          if mode is specified to be text ('r'), this defaults to 'utf-8'.
        cache: whether to attempt caching the resource. Defaults to True only if
          the given URL specifies a remote resource.
    Returns:
        All bytes form the specified resource, or a decoded string of those.
    """
    with read_handle(url, cache, mode=mode) as handle:
        data = handle.read()

    if encoding:
        data = data.decode(encoding)

    return data


@contextmanager
def read_handle(url, cache=None, mode="rb"):
    """Read from any URL with a file handle.

    Use this to get a handle to a file rather than eagerly load the data:

    ```
    with read_handle(url) as handle:
    result = something.load(handle)

    result.do_something()

    ```

    When program execution leaves this `with` block, the handle will be closed
    automatically.

    Args:
        url: a URL including scheme or a local path
    Returns:
        A file handle to the specified resource if it could be reached.
        The handle will be closed automatically once execution leaves this context.
    """
    url = scope_url(url)

    scheme = urlparse(url).scheme

    if cache == "purge":
        _purge_cached(url)
        cache = None

    if _is_remote(scheme) and cache is None:
        cache = True
        log.debug("Cache not specified, enabling because resource is remote.")

    if cache:
        handle = _read_and_cache(url, mode=mode)
    else:
        if scheme in ("http", "https"):
            handle = _handle_web_url(url, mode=mode)
        elif scheme in ("gs"):
            handle = _handle_gfile(url, mode=mode)
        else:
            handle = open(url, mode=mode)

    yield handle
    handle.close()


# Handlers


def _handle_gfile(url, mode="rb"):
    return GFile(url, mode)


def _handle_web_url(url, mode="r"):
    return request.urlopen(url)


# Helper Functions


def _is_remote(scheme):
    return scheme in ("http", "https", "gs")


RESERVED_PATH_CHARS = re.compile("[^a-zA-Z0-9]")
LUCID_CACHE_DIR_NAME = "lucid_cache"
MAX_FILENAME_LENGTH = 200
_LUCID_CACHE_DIR = None  # filled on first use


def local_cache_path(remote_url):
    global _LUCID_CACHE_DIR
    """Returns the path that remote_url would be cached at locally."""
    local_name = RESERVED_PATH_CHARS.sub("_", remote_url)
    if len(local_name) > MAX_FILENAME_LENGTH:
        filename_hash = hashlib.sha256(local_name.encode("utf-8")).hexdigest()
        truncated_name = (
            local_name[: (MAX_FILENAME_LENGTH - (len(filename_hash)) - 1)]
            + "-"
{'fraction_non_alphanumeric': 0.09798657718120805, 'fraction_numerical': 0.004250559284116331, 'mean_word_length': 3.221435316336166, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2939237', 'n_tokens_mistral': 2579, 'n_tokens_neox': 2471, 'n_words': 1009}