<commit_before>package edu.umass.cs.iesl.scalacommons

import _root_.javax.xml.parsers.SAXParser
import xml.factory.XMLLoader
import org.xml.sax.InputSource

import scala.xml._
import collection.TraversableOnce
import java.io.{InputStream, File}
import io.BufferedSource


object XmlUtils {
  def descendantExcluding(node: Node, excludeLabels: Seq[String]): List[Node] =
  {
    node.child.toList.filter(c => !excludeLabels.contains(c.label)).flatMap { x => x::descendantExcluding (x,excludeLabels) }
  }



  // http://stackoverflow.com/questions/8525675/how-to-get-a-streaming-iteratornode-from-a-large-xml-document
  def processRecordsStreaming[T](input: InputStream)(f: Node => T) {
    new scala.xml.parsing.ConstructingParser(new BufferedSource(input), false) {
      nextch // initialize per documentation
      document
      // trigger parsing by requesting document

      var depth = 0 // track depth

      override def elemStart(pos: Int, pre: String, label: String,
                             attrs: MetaData, scope: NamespaceBinding) {
        super.elemStart(pos, pre, label, attrs, scope)
        depth += 1
      }

      override def elemEnd(pos: Int, pre: String, label: String) {
        depth -= 1
        super.elemEnd(pos, pre, label)
      }

      override def elem(pos: Int, pre: String, label: String, attrs: MetaData,
                        pscope: NamespaceBinding, nodes: NodeSeq): NodeSeq = {
        val node: Node = super.elem(pos, pre, label, attrs, pscope, nodes).head
        depth match {
          case 1 => <dummy/> // dummy final roll up
          case 2 => f(node); NodeSeq.Empty // process and discard first-level nodes
          case _ => node // roll up other nodes
        }
      }
    }
  }

  // http://stackoverflow.com/questions/8525675/how-to-get-a-streaming-iteratornode-from-a-large-xml-document
  def processRecordsStreaming[T](input: InputStream, labels: Seq[String])(f: Node => T) {
    new scala.xml.parsing.ConstructingParser(new BufferedSource(input), false) {
      nextch // initialize per documentation
      document
      // trigger parsing by requesting document

      var depth = 0 // track depth


      override def elemStart(pos: Int, pre: String, label: String,
                             attrs: MetaData, scope: NamespaceBinding) {
        super.elemStart(pos, pre, label, attrs, scope)
        depth += 1
      }

      override def elemEnd(pos: Int, pre: String, label: String) {
        depth -= 1
        super.elemEnd(pos, pre, label)
      }

      override def elem(pos: Int, pre: String, label: String, attrs: MetaData,
                        pscope: NamespaceBinding, nodes: NodeSeq): NodeSeq = {
        val node: Node = super.elem(pos, pre, label, attrs, pscope, nodes).head
        if (labels contains label) {
          f(node); if (depth == 1) <dummy/> else NodeSeq.Empty
        } else node
        /*
        depth match {
          case 1 => <dummy/> // dummy final roll up
          case _ => if(labels contains label) {f(node); NodeSeq.Empty } else node
        }*/
      }
    }
  }

  // http://stackoverflow.com/questions/3797699/generator-block-to-iterator-stream-conversion
  def generatorToTraversable[T](func: (T => Unit) => Unit) = new Traversable[T] {
    def foreach[X](f: T => X) {
      func(f(_))
    }
  }

  /*
 The result of generatorToTraversable is not traversable more than once (even though a new ConstructingParser<commit_msg>Fix tagged citation parser for Gary Huang data<commit_after>package edu.umass.cs.iesl.scalacommons

import _root_.javax.xml.parsers.SAXParser
import xml.factory.XMLLoader
import org.xml.sax.InputSource

import scala.xml._
import collection.TraversableOnce
import java.io.{InputStream, File}
import io.BufferedSource


object XmlUtils {

  /**
   * All the text contained in a node, with no tags, but guaranteeing a space separating text bits that had a tag between them (e.g. blah<foo>bubba</foo>zimbob shourd give blah bubba zimbob, not blahbubbazimbob as node.text would have done.
   * @param node
   * @return
   */
  def spaceSeparatedText(node: Node): String = node.descendant.filter(_.isInstanceOf[Text]).map(_.text).mkString(" ").replaceAll("\\s|\"", " ").trim

  /* {
    // broken up for debugging
    val parts: Seq[String] = node.descendant.filter(_.isInstanceOf[Text]).map(_.text)
    val comb: String = parts.mkString(" ")
    val s: String = comb.replaceAll("\\s|\"", " ").trim
    s
  }*/

  def descendantExcluding(node: Node, excludeLabels: Seq[String]): List[Node] = {
    node.child.toList.filter(c => !excludeLabels.contains(c.label)).flatMap {
      x => x :: descendantExcluding(x, excludeLabels)
    }
  }


  // http://stackoverflow.com/questions/8525675/how-to-get-a-streaming-iteratornode-from-a-large-xml-document
  def processRecordsStreaming[T](input: InputStream)(f: Node => T) {
    new scala.xml.parsing.ConstructingParser(new BufferedSource(input), false) {
      nextch // initialize per documentation
      document
      // trigger parsing by requesting document

      var depth = 0 // track depth

      override def elemStart(pos: Int, pre: String, label: String,
                             attrs: MetaData, scope: NamespaceBinding) {
        super.elemStart(pos, pre, label, attrs, scope)
        depth += 1
      }

      override def elemEnd(pos: Int, pre: String, label: String) {
        depth -= 1
        super.elemEnd(pos, pre, label)
      }

      override def elem(pos: Int, pre: String, label: String, attrs: MetaData,
                        pscope: NamespaceBinding, nodes: NodeSeq): NodeSeq = {
        val node: Node = super.elem(pos, pre, label, attrs, pscope, nodes).head
        depth match {
          case 1 => <dummy/> // dummy final roll up
          case 2 => f(node); NodeSeq.Empty // process and discard first-level nodes
          case _ => node // roll up other nodes
        }
      }
    }
  }

  // http://stackoverflow.com/questions/8525675/how-to-get-a-streaming-iteratornode-from-a-large-xml-document
  def processRecordsStreaming[T](input: InputStream, labels: Seq[String])(f: Node => T) {
    new scala.xml.parsing.ConstructingParser(new BufferedSource(input), false) {
      nextch // initialize per documentation
      document
      // trigger parsing by requesting document

      var depth = 0 // track depth


      override def elemStart(pos: Int, pre: String, label: String,
                             attrs: MetaData, scope: NamespaceBinding) {
        super.elemStart(pos, pre, label, attrs, scope)
        depth += 1
      }

      override def elemEnd(pos: Int, pre: String, label: String) {
        depth -= 1
        super.elemEnd(pos, pre, label)
      }

      override def elem(pos: Int, pre: String, label: String, attrs: MetaData,
                        pscope: NamespaceBinding, nodes: NodeSeq): NodeSeq = {
        val node: Node = super.elem(pos, pre, label, attrs, pscope, nodes).head
        if (labels contains label) {
          f(node);
          if (depth == 1) <dummy/> else NodeSeq.Empty
        } else node
        /*
        depth match {
          case 1 => <dummy/> // dummy final roll up
          case _ => if(labels contains label) {f(node); NodeSeq.Empty } else node
        }*/
      }
    }
  }

  // http://stackoverflow.com/questions/3797699/generator-block-to-iterator-stream-conversion
  def generatorToTraversable[T](func: (T => Unit) => Unit) = new Traversable[T] {
    def foreach[X](f: T => X) {
      func(f(_))
    }
  }

  /*
 The result of generatorToTraversable is not traversable more than once (even though a new ConstructingParser
{'fraction_non_alphanumeric': 0.11849710982658959, 'fraction_numerical': 0.00814503415659485, 'mean_word_length': 2.5674789128397375, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3138118', 'n_tokens_mistral': 2359, 'n_tokens_neox': 2261, 'n_words': 624}