<commit_before>import binascii
import gzip
import json
import logging
import os
import glob
import fnmatch
import shutil
import time
import zlib
import io
import pickle
import tempfile
import re

from abc import ABC, abstractmethod
from collections import OrderedDict
from functools import total_ordering
from typing import Any, List, Optional, Iterable, Callable

import numpy
from azure.core import MatchConditions
from azure.core.exceptions import HttpResponseError
from azure.identity import DefaultAzureCredential
from azure.storage.blob import ContainerClient


from dpu_utils.utils.dataloading import save_json_gz, save_jsonl_gz

AZURE_PATH_PREFIX = "azure://"

__all__ = ['RichPath', 'LocalPath', 'AzurePath']

@total_ordering
class RichPath(ABC):
    """
    RichPath is an abstraction layer of local and remote paths allowing unified access
    of both local and remote files. Currently, only local and Azure blob paths are supported.
    
    To use Azure paths, if the current environment is set up you need no further action.

    See https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential for
    potential default configuration (az login, Service Principals, ManagedIdentity, etc.

    Alternatively a .json configuration file needs to be passed in the
    `RichPath.create()` function. The file has the format:
    
    ```
    {
        "storage_account_name": {
            "sas_token": "<PASSWORD>",
            "endpoint": "url if this is not a *.blob.core.windows.net endpoint"
            "cache_location": "optional location to cache blobs locally"<commit_msg>Set azure-blob logging level.<commit_after>import binascii
import gzip
import json
import logging
import os
import glob
import fnmatch
import shutil
import time
import zlib
import io
import pickle
import tempfile
import re

from abc import ABC, abstractmethod
from collections import OrderedDict
from functools import total_ordering
from typing import Any, List, Optional, Iterable, Callable

import numpy
from azure.core import MatchConditions
from azure.core.exceptions import HttpResponseError
from azure.identity import DefaultAzureCredential
from azure.storage.blob import ContainerClient


from dpu_utils.utils.dataloading import save_json_gz, save_jsonl_gz

AZURE_PATH_PREFIX = "azure://"

__all__ = ['RichPath', 'LocalPath', 'AzurePath']

logging.getLogger('azure.storage.blob').setLevel(logging.ERROR)
logging.getLogger('azure.core').setLevel(logging.ERROR)

@total_ordering
class RichPath(ABC):
    """
    RichPath is an abstraction layer of local and remote paths allowing unified access
    of both local and remote files. Currently, only local and Azure blob paths are supported.
    
    To use Azure paths, if the current environment is set up you need no further action.

    See https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential for
    potential default configuration (az login, Service Principals, ManagedIdentity, etc.

    Alternatively a .json configuration file needs to be passed in the
    `RichPath.create()` function. The file has the format:
    
    ```
    {
        "storage_account_name": {
            "sas_token": "<PASSWORD>",
            "endpoint": "url if this is not a *.blob.core.windows.net endpoint"
            "cache_location": "optional location to cache blobs locally"
{'fraction_non_alphanumeric': 0.06820194862710363, 'fraction_numerical': 0.0, 'mean_word_length': 4.212307692307692, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1766511', 'n_tokens_mistral': 928, 'n_tokens_neox': 899, 'n_words': 352}