<commit_before>from .compat import texttype


class Node(object):

    def __init__(self, location=None):  # kwarg-only
        self.location = location

    @classmethod
    def typed(cls, _type_, *args, **kwargs):
        node = cls(*args, **kwargs)
        node.__type__ = _type_
        return node

    def accept(self, visitor):
        raise NotImplementedError


class Symbol(Node):

    def __init__(self, name, **kw):
        self.name = name
        head, sep, tail = name.partition('/')
        if sep:
            self.ns, self.rel = head, tail
        else:
            self.ns, self.rel = None, name
        super(Symbol, self).__init__(**kw)

    def __repr__(self):
        return self.name

    def accept(self, visitor):
        return visitor.visit_symbol(self)


class String(Node):

    def __init__(self, value, **kw):
        self.value = texttype(value)
        super(String, self).__init__(**kw)

    def __repr__(self):
        return '"{}"'.format(
            repr(self.value).lstrip('u')[1:-1]
            .replace('\\"', '"').replace("\\'", "'")
            .replace('"', '\\"')
        )

    def accept(self, visitor):
        return visitor.visit_string(self)


class Number(Node):

    def __init__(self, value, **kw):
        self.value = value
        super(Number, self).__init__(**kw)
<commit_msg>Using json library to represent String value<commit_after>from json.encoder import encode_basestring_ascii

from .compat import texttype


class Node(object):

    def __init__(self, location=None):  # kwarg-only
        self.location = location

    @classmethod
    def typed(cls, _type_, *args, **kwargs):
        node = cls(*args, **kwargs)
        node.__type__ = _type_
        return node

    def accept(self, visitor):
        raise NotImplementedError


class Symbol(Node):

    def __init__(self, name, **kw):
        self.name = name
        head, sep, tail = name.partition('/')
        if sep:
            self.ns, self.rel = head, tail
        else:
            self.ns, self.rel = None, name
        super(Symbol, self).__init__(**kw)

    def __repr__(self):
        return self.name

    def accept(self, visitor):
        return visitor.visit_symbol(self)


class String(Node):

    def __init__(self, value, **kw):
        self.value = texttype(value)
        super(String, self).__init__(**kw)

    def __repr__(self):
        return encode_basestring_ascii(self.value)

    def accept(self, visitor):
        return visitor.visit_string(self)


class Number(Node):

    def __init__(self, value, **kw):
        self.value = value
        super(Number, self).__init__(**kw)

{'fraction_non_alphanumeric': 0.13188901558342836, 'fraction_numerical': 0.0007601672367920942, 'mean_word_length': 2.357142857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4575088', 'n_tokens_mistral': 820, 'n_tokens_neox': 792, 'n_words': 200}