<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: <NAME> <<EMAIL>>

import sys
import os
import re
from lxml import etree, objectify
from networkx import MultiDiGraph, write_gpickle

INTEGER_RE = re.compile('([0-9]+)')

class TigerDocumentGraph(MultiDiGraph):
    """
    A directed graph (networkx.MultiDiGraph) that represents all the
    sentences contained in a TigerXML file. A ``TigerDocumentGraph``
    contains a document root node (whose ID is stored in ``self.root``),
    which has an outgoing edge to the sentence root nodes of each
    sentence.

    Attributes
    ----------
    filename : str
        filename of the TigerXML file that was parsed
    corpus_id : str
        ID of the TigerXML document specified in the 'id' attribute
        of the <corpus> element
    root : str
        the ID of the root node of the document graph
    sentences : list of str
        sorted list of all sentence root node IDs (of sentences
        contained in this document graph)
    """
    def __init__(self, tiger_filepath):
        """
        Creates a directed graph that represents all syntax annotated
        sentences in the given TigerXML file.

        Parameters
        ----------
        tiger_filepath : str
            absolute or relative path to the TigerXML file to be parsed
        """
        # super calls __init__() of base class MultiDiGraph
        super(TigerDocumentGraph, self).__init__()

        utf8_parser = etree.XMLParser(encoding="utf-8")
        tigerxml_tree = etree.parse(tiger_filepath, utf8_parser)
        tigerxml_root = tigerxml_tree.getroot()

        self.filename = os.path.basename(tiger_filepath)
        self.corpus_id = tigerxml_root.attrib['id']

        # add root node of TigerDocumentGraph
        self.root = 'tiger_root_node'
        self.add_node(self.root, type='tiger_root_node')

        self.sentences = []
        for sentence in tigerxml_root.iterfind('./body/s'):
            self.__add_sentence_to_document(sentence)
        self.sentences = sorted(self.sentences, key=_natural_sort_key)<commit_msg>documentation: how to print all tokens contained in a Tiger docgraph.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: <NAME> <<EMAIL>>

import sys
import os
import re
from lxml import etree, objectify
from networkx import MultiDiGraph, write_gpickle

INTEGER_RE = re.compile('([0-9]+)')

class TigerDocumentGraph(MultiDiGraph):
    """
    A directed graph (networkx.MultiDiGraph) that represents all the
    sentences contained in a TigerXML file. A ``TigerDocumentGraph``
    contains a document root node (whose ID is stored in ``self.root``),
    which has an outgoing edge to the sentence root nodes of each
    sentence.

    Attributes
    ----------
    filename : str
        filename of the TigerXML file that was parsed
    corpus_id : str
        ID of the TigerXML document specified in the 'id' attribute
        of the <corpus> element
    root : str
        the ID of the root node of the document graph
    sentences : list of str
        sorted list of all sentence root node IDs (of sentences
        contained in this document graph)

    The attribute dict of each sentence root node contains a key
    'tokens', which maps to a sorted list of token node IDs (str). To
    print all tokens of a Tiger document, just do:

    tdg = TigerDocumentGraph('/path/to/tiger.file')
    for sentence_root_node in tdg.sentences:
        for token_node_id in tdg.node[sentence_root_node]['tokens']:
            print tdg.node[token_node_id]['word']
    """
    def __init__(self, tiger_filepath):
        """
        Creates a directed graph that represents all syntax annotated
        sentences in the given TigerXML file.

        Parameters
        ----------
        tiger_filepath : str
            absolute or relative path to the TigerXML file to be parsed
        """
        # super calls __init__() of base class MultiDiGraph
        super(TigerDocumentGraph, self).__init__()

        utf8_parser = etree.XMLParser(encoding="utf-8")
        tigerxml_tree = etree.parse(tiger_filepath, utf8_parser)
        tigerxml_root = tigerxml_tree.getroot()

        self.filename = os.path.basename(tiger_filepath)
        self.corpus_id = tigerxml_root.attrib['id']

        # add root node of TigerDocumentGraph
        self.root = 'tiger_root_node'
        self.add_node(self.root, type='tiger_root_node')

        self.sentences = []
        for sentence in tigerxml_root.iterfind('./body/s'):
            self.__add_sentence_to_document(sentence)
        self.sentences = sorted(self.sentences, key=_natural_sort_key)
{'fraction_non_alphanumeric': 0.08609556607834697, 'fraction_numerical': 0.002582866982350409, 'mean_word_length': 2.6998407643312103, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '340497', 'n_tokens_mistral': 1344, 'n_tokens_neox': 1295, 'n_words': 444}