<commit_before>import os
from azure.datalake.store import lib, core, multithread
import re


class ADL(object):
    def __init__(self):
        self.token = None

    @classmethod
    def _split_url(cls, url):
        match = re.match(r'adl://(.*)\.azuredatalakestore\.net\/(.*)$', url)
        if not match:
            raise Exception("Invalid ADL url '{0}'".format(url))
        else:
            return (match.group(1), match.group(2))

    def _get_token(self):
        if self.token is None:
            self.token = lib.auth()
        return self.token

    def _create_adapter(self, store_name):
        return core.AzureDLFileSystem(self._get_token(), store_name=store_name)<commit_msg>Update adl.py<commit_after>import re
from azure.datalake.store import core, lib


class ADL(object):
    def __init__(self):
        self.token = None

    @classmethod
    def _split_url(cls, url):
        match = re.match(r'adl://(.*)\.azuredatalakestore\.net\/(.*)$', url)
        if not match:
            raise Exception("Invalid ADL url '{0}'".format(url))
        else:
            return (match.group(1), match.group(2))

    def _get_token(self):
        if self.token is None:
            self.token = lib.auth()
        return self.token

    def _create_adapter(self, store_name):
        return core.AzureDLFileSystem(self._get_token(), store_name=store_name)
{'fraction_non_alphanumeric': 0.13167520117044623, 'fraction_numerical': 0.0043891733723482075, 'mean_word_length': 2.5440414507772022, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2047065', 'n_tokens_mistral': 448, 'n_tokens_neox': 434, 'n_words': 99}