<commit_before>import Foundation


/// Text nodes.  Cannot have child nodes (for now, not sure if we will need them).
///
class TextNode: Node {

    var contents: String

    // MARK: - CustomReflectable
    
    override public var customMirror: Mirror {
        get {
            return Mirror(self, children: ["type": "text", "name": name, "text": contents, "parent": parent.debugDescription], ancestorRepresentation: .suppressed)
        }
    }
    
    // MARK: - Initializers
    
    init(text: String) {
        contents = text

        super.init(name: "text")
    }

    /// Node length.
    ///
    func length() -> Int {
        return contents.characters.count
    }

    // MARK: - Node

    /// Checks if the specified node requires a closing paragraph separator.
    ///
    func needsClosingParagraphSeparator() -> Bool {
        guard length() > 0 else {
            return false
        }

        guard !hasRightBlockLevelSibling() else {
            return true
        }

        return !isLastInTree() && isLastInAncestorEndingInBlockLevelSeparation()
    }

    // MARK: - LeafNode
    
    func text() -> String {
        return contents
    }

    // MARK - Hashable

    override public var hashValue: Int {
        return name.hashValue ^ contents.hashValue
    }

    // MARK: - Node Equatable

    override func isEqual(other: Node) -> Bool {
        guard let textNode = other as? TextNode else {
            return false
        }
        return self.name == textNode.name && self.contents == textNode.contents
    }    
}
<commit_msg>TextNode: Fixing merge glitch<commit_after>import Foundation


/// Text nodes.  Cannot have child nodes (for now, not sure if we will need them).
///
class TextNode: Node {

    var contents: String

    // MARK: - CustomReflectable
    
    override public var customMirror: Mirror {
        get {
            return Mirror(self, children: ["type": "text", "name": name, "text": contents, "parent": parent.debugDescription], ancestorRepresentation: .suppressed)
        }
    }
    
    // MARK: - Initializers
    
    init(text: String) {
        contents = text

        super.init(name: "text")
    }

    /// Node length.
    ///
    func length() -> Int {
        return contents.characters.count
    }

    // MARK: - Node

    /// Checks if the specified node requires a closing paragraph separator.
    ///
    func needsClosingParagraphSeparator() -> Bool {
        guard length() > 0 else {
            return false
        }

        guard !hasRightBlockLevelSibling() else {
            return true
        }

        return !isLastInTree() && isLastInAncestorEndingInBlockLevelSeparation()
    }

    // MARK: - LeafNode
    
    func text() -> String {
        return contents
    }

    // MARK - Hashable

    override public var hashValue: Int {
        return name.hashValue ^ contents.hashValue
    }

    // MARK: - Equatable

    override public func isEqual(_ object: Any?) -> Bool {
        guard let textNode = object as? TextNode else {
            return false
        }
        return self.name == textNode.name && self.contents == textNode.contents
    }    
}

{'fraction_non_alphanumeric': 0.10379746835443038, 'fraction_numerical': 0.0006329113924050633, 'mean_word_length': 1.8920402561756633, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5315875', 'n_tokens_mistral': 908, 'n_tokens_neox': 851, 'n_words': 260}