<commit_before>package edu.cmu.lti.nlp.amr

import scala.collection.mutable.Map
import scala.collection.mutable.Set
import scala.collection.mutable.ArrayBuffer
import scala.collection.immutable.Queue
import scala.collection.immutable
import scala.util.parsing.combinator._

case class Graph(var root: Node, spans: ArrayBuffer[Span], getNodeById: Map[String, Node], getNodeByName: Map[String, Node]) {

    def duplicate : Graph = {
        // Makes a copy of the graph
        // Assumes that getNodeById exists and is properly set up (for 'nodes' call)
        val getNodeById2 : Map[String, Node] = Map()
        for (node <- nodes) {
            val Node(id, name, concept, relations, topologicalOrdering, variableRelations, alignment, spans) = node
            getNodeById2(id) = Node(id, name, concept, List(), List(), List(), alignment, spans)
        }
        for (node <- nodes) {
            val node2 = getNodeById2(node.id)
            node2.relations = node.relations.map(x => (x._1, getNodeById2(x._2.id)))
            node2.topologicalOrdering = node.topologicalOrdering.map(x => (x._1, getNodeById2(x._2.id)))
            node2.variableRelations = node.variableRelations.map(x => (x._1, Var(getNodeById2(x._2.node.id), x._2.name)))
        }
        val getNodeByName2 = getNodeByName.map(x => (x._1, getNodeById2(x._2.id)))
        logger(1, "getNodeById = " + getNodeById)
        logger(1, "getNodeById2 = " + getNodeById2)
        val root2 = if (getNodeById2.contains(root.id)) {
            getNodeById2(root.id)
        } else {
            Graph.empty().root  // sometimes the root is not in the spans (if it is from automatically aligned spans, and the root is not in the spans).  So we create a dummy root which will get re-assigned later
        }
        return Graph(root2, spans.clone, getNodeById2, getNodeByName2)
    }

    def clearUnalignedNodes() {
        // Removes all the unaligned nodes from the graph (useful for oracle experiments)
        // WARNING: this can break the topological ordering
        for (node <- nodes) {
            if (node.spans.size == 0) { // Unaligned
                logger(1, "clearUnalignedNodes():  removing unaligned node: "+node.name+" / "+node.concept)
                getNodeById -= node.id
                if (node.name != None) {
                    getNodeByName -= node.name.get
                }
            }
        }
        if (nodes.filter(x => !getNodeById.contains(x.id)).size  > 0) {
            logger(1, "clearUnalignedNodes() deleting relations to the following nodes: "+nodes.filter(x => !getNodeById.contains(x.id)).toList)
        }
        for (node <- nodes) {
            node.topologicalOrdering = node.topologicalOrdering.filter(x => getNodeById.contains(x._2.id)) // Warning this may break the topological ordering
            node.relations = node.relations.filter(x => getNodeById.contains(x._2.id))
            node.variableRelations = node.variableRelations.filter(x => getNodeById.contains(x._2.node.id))
        }
    }

    def clearEdges() : Graph = {
        // Initializes the graph from the spans (effectively clearing the edges)
        // Sets relations, but leaves topologicalOrdering and variableRelations blank
        // As a side effect, it re-assigns ids to all the nodes in the graph.  TODO: why did I do it this way???  Couldn't I just keep the ids the same???  Maybe it was so I could simulate what it is like to have ids from stage 1
        // TODO: CHANGE THIS SO IT DOESN'T RE-ASSIGN IDS

        var currentId = 0
        getNodeById.clear
        getNodeByName.clear

        def addRec(node: Node) : (Node, List[String]) = {
            var ids = List(currentId.toString)
            //val node2 = Node(node.id, node.name, node.concept, List(), List(), List(), node.alignment, node.spans)
            val node2 = Node(currentId.toString, node.name, node.concept, List(), List(), List(), node.alignment, node.spans)   // TODO: is this bugfix correct???
            var relations = List[(String, Node)]()
            getNodeById(currentId.toString) = node2
            node.name match {
                case Some(name) => { getNodeByName(name) = node2 }
                case None => Unit
            }
            currentId += 1
            for ((relation, child) <- node.topologicalOrdering) {
                val result = addRec(child)
                ids = ids ::: result._2
                relations = (relation, result._1) :: relations
            }
            node2.relations = relations.reverse
            return (node2, ids)
        }

        // Add all the non-coref spans
        for (span <- spans if !span.coRef) {
            span.nodeIds = addRec(span.amr)._2
        }

        // Set the span ids for the coref spans correctly
        for (span <- spans if span.coRef) {
            span.nodeIds = List()
        }

        for ((id, node) <- getNodeById) {
            for { spanIndex <- node.spans
                  span = spans(spanIndex)
                  if span.coRef
                } {
                span.nodeIds = span.nodeIds ::: List(id)
            }
        }
        return this
    }

    def printRoot : String = {
        return "# ::root\t" + root.id + "\t" + root.concept
    }

    def printNodes : List[String] = {
        nodes.map(node =>
            if (node.spans.size > 0) {
                node.id + "\t" + node.nameStr + "\t" + node.concept + "\t" + spans(node.spans(0)).start + "-" + spans(node.spans(0)).end
            } else {
                node.id + "\t" + node.nameStr + "\t" + node.concept + "\t"
            }
        ).toList.sorted
    }

    def printEdges : List[String] = {
        var edges : List[String] = List()
        val Relation = """:?(.*)""".r

        for { node1 <- nodes
              (label, node2) <- node1.relations
              Relation(relation) = label    // label includes the ":"
            } {
                edges = node1.id + "\t" + node1.concept + "\t" + relation + "\t" + node2.id + "\t" + node2.concept :: edges
        }
        return edges.sorted
    }<commit_msg>Change format of edges (make it more human-reable)<commit_after>package edu.cmu.lti.nlp.amr

import scala.collection.mutable.Map
import scala.collection.mutable.Set
import scala.collection.mutable.ArrayBuffer
import scala.collection.immutable.Queue
import scala.collection.immutable
import scala.util.parsing.combinator._

case class Graph(var root: Node, spans: ArrayBuffer[Span], getNodeById: Map[String, Node], getNodeByName: Map[String, Node]) {

    def duplicate : Graph = {
        // Makes a copy of the graph
        // Assumes that getNodeById exists and is properly set up (for 'nodes' call)
        val getNodeById2 : Map[String, Node] = Map()
        for (node <- nodes) {
            val Node(id, name, concept, relations, topologicalOrdering, variableRelations, alignment, spans) = node
            getNodeById2(id) = Node(id, name, concept, List(), List(), List(), alignment, spans)
        }
        for (node <- nodes) {
            val node2 = getNodeById2(node.id)
            node2.relations = node.relations.map(x => (x._1, getNodeById2(x._2.id)))
            node2.topologicalOrdering = node.topologicalOrdering.map(x => (x._1, getNodeById2(x._2.id)))
            node2.variableRelations = node.variableRelations.map(x => (x._1, Var(getNodeById2(x._2.node.id), x._2.name)))
        }
        val getNodeByName2 = getNodeByName.map(x => (x._1, getNodeById2(x._2.id)))
        logger(1, "getNodeById = " + getNodeById)
        logger(1, "getNodeById2 = " + getNodeById2)
        val root2 = if (getNodeById2.contains(root.id)) {
            getNodeById2(root.id)
        } else {
            Graph.empty().root  // sometimes the root is not in the spans (if it is from automatically aligned spans, and the root is not in the spans).  So we create a dummy root which will get re-assigned later
        }
        return Graph(root2, spans.clone, getNodeById2, getNodeByName2)
    }

    def clearUnalignedNodes() {
        // Removes all the unaligned nodes from the graph (useful for oracle experiments)
        // WARNING: this can break the topological ordering
        for (node <- nodes) {
            if (node.spans.size == 0) { // Unaligned
                logger(1, "clearUnalignedNodes():  removing unaligned node: "+node.name+" / "+node.concept)
                getNodeById -= node.id
                if (node.name != None) {
                    getNodeByName -= node.name.get
                }
            }
        }
        if (nodes.filter(x => !getNodeById.contains(x.id)).size  > 0) {
            logger(1, "clearUnalignedNodes() deleting relations to the following nodes: "+nodes.filter(x => !getNodeById.contains(x.id)).toList)
        }
        for (node <- nodes) {
            node.topologicalOrdering = node.topologicalOrdering.filter(x => getNodeById.contains(x._2.id)) // Warning this may break the topological ordering
            node.relations = node.relations.filter(x => getNodeById.contains(x._2.id))
            node.variableRelations = node.variableRelations.filter(x => getNodeById.contains(x._2.node.id))
        }
    }

    def clearEdges() : Graph = {
        // Initializes the graph from the spans (effectively clearing the edges)
        // Sets relations, but leaves topologicalOrdering and variableRelations blank
        // As a side effect, it re-assigns ids to all the nodes in the graph.  TODO: why did I do it this way???  Couldn't I just keep the ids the same???  Maybe it was so I could simulate what it is like to have ids from stage 1
        // TODO: CHANGE THIS SO IT DOESN'T RE-ASSIGN IDS

        var currentId = 0
        getNodeById.clear
        getNodeByName.clear

        def addRec(node: Node) : (Node, List[String]) = {
            var ids = List(currentId.toString)
            //val node2 = Node(node.id, node.name, node.concept, List(), List(), List(), node.alignment, node.spans)
            val node2 = Node(currentId.toString, node.name, node.concept, List(), List(), List(), node.alignment, node.spans)   // TODO: is this bugfix correct???
            var relations = List[(String, Node)]()
            getNodeById(currentId.toString) = node2
            node.name match {
                case Some(name) => { getNodeByName(name) = node2 }
                case None => Unit
            }
            currentId += 1
            for ((relation, child) <- node.topologicalOrdering) {
                val result = addRec(child)
                ids = ids ::: result._2
                relations = (relation, result._1) :: relations
            }
            node2.relations = relations.reverse
            return (node2, ids)
        }

        // Add all the non-coref spans
        for (span <- spans if !span.coRef) {
            span.nodeIds = addRec(span.amr)._2
        }

        // Set the span ids for the coref spans correctly
        for (span <- spans if span.coRef) {
            span.nodeIds = List()
        }

        for ((id, node) <- getNodeById) {
            for { spanIndex <- node.spans
                  span = spans(spanIndex)
                  if span.coRef
                } {
                span.nodeIds = span.nodeIds ::: List(id)
            }
        }
        return this
    }

    def printRoot : String = {
        return "# ::root\t" + root.id + "\t" + root.concept
    }

    def printNodes : List[String] = {
        nodes.map(node =>
            if (node.spans.size > 0) {
                //node.id + "\t" + node.nameStr + "\t" + node.concept + "\t" + spans(node.spans(0)).start + "-" + spans(node.spans(0)).end
                node.id + "\t" + node.concept + "\t" + spans(node.spans(0)).start + "-" + spans(node.spans(0)).end
            } else {
                //node.id + "\t" + node.nameStr + "\t" + node.concept + "\t"
                node.id + "\t" + node.concept + "\t"
            }
        ).toList.sorted
    }

    def printEdges : List[String] = {
        var edges : List[String] = List()
        val Relation = """:?(.*)""".r

        for { node1 <- nodes
              (label, node2) <- node1.relations
              Relation(relation) = label    // label includes the ":"
            } {
                //edges = node1.id + "\t" + node1.concept + "\t" + relation + "\t" + node2.id + "\t" + node2.concept :: edges
                edges = node1.concept + "\t" + relation + "\t" + node2.concept + "\t" + node1.id + "\t" +  node2.id + "\t" :: edges
        }
        return edges.sorted
    }
{'fraction_non_alphanumeric': 0.12526963329871374, 'fraction_numerical': 0.01006630981864664, 'mean_word_length': 1.9572407276163477, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3573774', 'n_tokens_mistral': 3814, 'n_tokens_neox': 3652, 'n_words': 999}