<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import sys
import math
import random
import logging


class DharmaConst(object):
    """Configuration settings for the Dharma generator."""
    pass


class MetaBlock(object):
    """Grammar extension which loads code fragments from a file into the grammar."""

    def __init__(self, path, parent):
        self.parent = parent
        path = os.path.expanduser(path)
        if os.path.exists(path):
            with open(path) as fo:
                self.content = fo.read()
        else:
            logging.warning('%s: Unable to load resource for block() "%s"', parent.id(), path)
            self.content = path

    def generate(self, state):
        return self.content


class MetaURI(object):
    """Grammar extension which loads a random file URI into the generated code."""

    def __init__(self, path, parent):
        self.parent = parent
        if path in DharmaConst.URI_TABLE:
            path = DharmaConst.URI_TABLE[path]
        if os.path.isdir(path):
            self.path = [p for p in (os.path.join(path, f) for f in os.listdir(path)) if os.path.isfile(p)]
        elif os.path.exists(path):
            self.path = [path]
        else:
            logging.warning('%s: Unable to identify argument of uri() "%s"', parent.id(), path)
            self.path = [path]

    def generate(self, state):
        return random.choice(self.path)

<commit_msg>Use os.path.expanduser for evaluating the path<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import sys
import math
import random
import logging


class DharmaConst(object):
    """Configuration settings for the Dharma generator."""
    pass


class MetaBlock(object):
    """Grammar extension which loads code fragments from a file into the grammar."""

    def __init__(self, path, parent):
        self.parent = parent
        path = os.path.expanduser(path)
        if os.path.exists(path):
            with open(path) as fo:
                self.content = fo.read()
        else:
            logging.warning('%s: Unable to load resource for block() "%s"', parent.id(), path)
            self.content = path

    def generate(self, state):
        return self.content


class MetaURI(object):
    """Grammar extension which loads a random file URI into the generated code."""

    def __init__(self, path, parent):
        self.parent = parent
        if path in DharmaConst.URI_TABLE:
            path = DharmaConst.URI_TABLE[path]
        path = os.path.expanduser(path)
        if os.path.isdir(path):
            self.path = [p for p in (os.path.join(path, f) for f in os.listdir(path)) if os.path.isfile(p)]
        elif os.path.exists(path):
            self.path = [path]
        else:
            logging.warning('%s: Unable to identify argument of uri() "%s"', parent.id(), path)
            self.path = [path]

    def generate(self, state):
        return random.choice(self.path)


{'fraction_non_alphanumeric': 0.10907424381301559, 'fraction_numerical': 0.002444240757714635, 'mean_word_length': 2.457233368532207, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '768268', 'n_tokens_mistral': 965, 'n_tokens_neox': 924, 'n_words': 337}