<commit_before>package org.wquery.loader
import javax.xml.parsers.SAXParserFactory
import java.io.File
import org.wquery.utils.Logging
import org.xml.sax.{Locator, Attributes}
import org.xml.sax.helpers.DefaultHandler
import scala.collection.mutable.{Set, Map, ListBuffer}
import org.wquery.model._
import org.wquery.WQueryUpdateBreaksRelationPropertyException

class GridLoader extends WordNetLoader with Logging {
  override def canLoad(url: String): Boolean = url.endsWith(".xml") // TODO provide a better check

  override def load(url: String, wordNet: WordNet) = {
    val factory = SAXParserFactory.newInstance

    factory.newSAXParser.parse(new File(url), new GridHandler(wordNet))
    info("WordNet loaded via GridLoader")
  }

}

class GridHandler(wordnet: WordNet) extends DefaultHandler with Logging {
  // parser attributes
  private var locator: Locator = null
  
  // global attributes
  private val synsetsById = Map[String, Synset]()
  private val ilrRelationsTuples = new ListBuffer[(Synset, String, String)]()    
  private val genericRelationsTuples = new ListBuffer[(Synset, String, String)]()
  
  // per synset attributes
  private var synsetId: String = null
  private var synsetPos: String = null
  private var synsetIlrRelationsTuples: ListBuffer[(String, String)] = null    
  private var synsetGenericRelationsTuples: ListBuffer[(String, String)] = null
  private var synsetSenses: ListBuffer[(String, String)] = null 
  
  // per subtag attributes
  private var literalSense: String = null  
  private var ilrType: String = null
  private var text = new StringBuilder
  private var previousText: StringBuilder = null

  override def setDocumentLocator(loc: Locator) { locator = loc } 
    
  override def startElement(uri: String, localName: String, tagName: String, attributes: Attributes) {
    pushText       
        
    tagName match {
      case "SYNSET" =>
        synsetId = null
        synsetPos = null    
        synsetIlrRelationsTuples = new ListBuffer[(String, String)]    
        synsetGenericRelationsTuples = new ListBuffer[(String, String)]        
        synsetSenses = new ListBuffer[(String, String)]        
      case "LITERAL" =>
        literalSense = attributes.getValue("sense")
      case "ILR" =>
      ilrType = attributes.getValue("type")
      case _ =>
        // skip
    }    
  }
    
  override def characters(chars: Array[Char], start: Int, length: Int) {      
    text.appendAll(chars, start, length)
  }

  override def endElement(uri: String, localName: String, tagName: String) {
    tagName match {
      case "SYNSET" =>
        endSynsetTag
      case tagName =>
        val content = text.toString.trim
        
        tagName match {
          case "ID" =>
            synsetId = content
          case "POS" =>
            synsetPos = content            
            endGenericTag("POS", content)            
          case "LITERAL" =>
            synsetSenses += ((content, literalSense))
            literalSense = null
          case "SENSE" =>
            literalSense = content
          case "ILR" =>
            synsetIlrRelationsTuples += ((ilrType, content))    
            ilrType = null
          case "TYPE" =>
            ilrType = content
          case tagName =>
            endGenericTag(tagName, content)          
        }
    }
      
    tagName match {
      case "SENSE" =>
        popText
      case _ =>
        pushText
    }        
  }

  private def pushText {
    previousText = text
    text = new StringBuilder        
  }
    
  private def popText {
    text = previousText
  }    
  
  private def endSynsetTag {    
    if (synsetId == null) {
      locatedWarn("SYNSET tag does not contain ID subtag")
    } else {
      if (synsetPos == null) {
        locatedWarn("SYNSET '" + synsetId + "' does not contain POS tag - 'x' symbol used as the POS tag value")
        synsetPos = "x"
      }
      
      val senses = new ListBuffer[Sense]
      
      synsetSenses.toList.zipWithIndex.foreach {case ((content, literalSense), index) =>
        try {
          if (literalSense != null) {
            senses += new Sense(content, literalSense.toInt, synsetPos)
          } else {
            warnInvalidSubtag("LITERAL", content, "sense")
          }
        } catch {
          case ex: NumberFormatException =>
            warnInvalidSubtag("LITERAL", content, "sense") 
        }
      }

      val synset = wordnet.store.addSynset(Some(synsetId), senses.toList, Nil, moveSenses = false)
      synsetsById(synset.id) = synset
      
      for ((ilrType, content) <- synsetIlrRelationsTuples) {
        if (ilrType == null || ilrType.trim.isEmpty) {
          warnInvalidSubtag("ILR", content, "type")      
        } else {
          ilrRelationsTuples += ((synset, ilrType.trim, content))
        }        
      }
      
      for ((tagName, content) <- synsetGenericRelationsTuples) {
        genericRelationsTuples += ((synset, tagName, content))
      }      
    }
  }  

  private def endGenericTag(tagName: String, content: String) {
    tagName.toLowerCase match {
      case "synonym" =>
        // skip
      case "word" =>
        // skip
      case "valency" =>
        //  skip
      case relName =>
        synsetGenericRelationsTuples += ((relName, content))
    }
  }
    
  override def endDocument {
    info("Synsets loaded") 
    createIlrRelations    <commit_msg>'related' relation for empty ILR types<commit_after>package org.wquery.loader
import javax.xml.parsers.SAXParserFactory
import java.io.File
import org.wquery.utils.Logging
import org.xml.sax.{Locator, Attributes}
import org.xml.sax.helpers.DefaultHandler
import scala.collection.mutable.{Set, Map, ListBuffer}
import org.wquery.model._
import org.wquery.WQueryUpdateBreaksRelationPropertyException

class GridLoader extends WordNetLoader with Logging {
  override def canLoad(url: String): Boolean = url.endsWith(".xml") // TODO provide a better check

  override def load(url: String, wordNet: WordNet) = {
    val factory = SAXParserFactory.newInstance

    factory.newSAXParser.parse(new File(url), new GridHandler(wordNet))
    info("WordNet loaded via GridLoader")
  }

}

class GridHandler(wordnet: WordNet) extends DefaultHandler with Logging {
  // parser attributes
  private var locator: Locator = null
  
  // global attributes
  private val synsetsById = Map[String, Synset]()
  private val ilrRelationsTuples = new ListBuffer[(Synset, String, String)]()    
  private val genericRelationsTuples = new ListBuffer[(Synset, String, String)]()
  
  // per synset attributes
  private var synsetId: String = null
  private var synsetPos: String = null
  private var synsetIlrRelationsTuples: ListBuffer[(String, String)] = null    
  private var synsetGenericRelationsTuples: ListBuffer[(String, String)] = null
  private var synsetSenses: ListBuffer[(String, String)] = null 
  
  // per subtag attributes
  private var literalSense: String = null  
  private var ilrType: String = null
  private var text = new StringBuilder
  private var previousText: StringBuilder = null

  override def setDocumentLocator(loc: Locator) { locator = loc } 
    
  override def startElement(uri: String, localName: String, tagName: String, attributes: Attributes) {
    pushText       
        
    tagName match {
      case "SYNSET" =>
        synsetId = null
        synsetPos = null    
        synsetIlrRelationsTuples = new ListBuffer[(String, String)]    
        synsetGenericRelationsTuples = new ListBuffer[(String, String)]        
        synsetSenses = new ListBuffer[(String, String)]        
      case "LITERAL" =>
        literalSense = attributes.getValue("sense")
      case "ILR" =>
      ilrType = attributes.getValue("type")
      case _ =>
        // skip
    }    
  }
    
  override def characters(chars: Array[Char], start: Int, length: Int) {      
    text.appendAll(chars, start, length)
  }

  override def endElement(uri: String, localName: String, tagName: String) {
    tagName match {
      case "SYNSET" =>
        endSynsetTag
      case tagName =>
        val content = text.toString.trim
        
        tagName match {
          case "ID" =>
            synsetId = content
          case "POS" =>
            synsetPos = content            
            endGenericTag("POS", content)            
          case "LITERAL" =>
            synsetSenses += ((content, literalSense))
            literalSense = null
          case "SENSE" =>
            literalSense = content
          case "ILR" =>
            synsetIlrRelationsTuples += ((ilrType, content))    
            ilrType = null
          case "TYPE" =>
            ilrType = content
          case tagName =>
            endGenericTag(tagName, content)          
        }
    }
      
    tagName match {
      case "SENSE" =>
        popText
      case _ =>
        pushText
    }        
  }

  private def pushText {
    previousText = text
    text = new StringBuilder        
  }
    
  private def popText {
    text = previousText
  }    
  
  private def endSynsetTag {    
    if (synsetId == null) {
      locatedWarn("SYNSET tag does not contain ID subtag")
    } else {
      if (synsetPos == null) {
        locatedWarn("SYNSET '" + synsetId + "' does not contain POS tag - 'x' symbol used as the POS tag value")
        synsetPos = "x"
      }
      
      val senses = new ListBuffer[Sense]
      
      synsetSenses.toList.zipWithIndex.foreach {case ((content, literalSense), index) =>
        try {
          if (literalSense != null) {
            senses += new Sense(content, literalSense.toInt, synsetPos)
          } else {
            warnInvalidSubtag("LITERAL", content, "sense")
          }
        } catch {
          case ex: NumberFormatException =>
            warnInvalidSubtag("LITERAL", content, "sense") 
        }
      }

      val synset = wordnet.store.addSynset(Some(synsetId), senses.toList, Nil, moveSenses = false)
      synsetsById(synset.id) = synset
      
      for ((ilrType, content) <- synsetIlrRelationsTuples) {
        val ilrName = if (ilrType == null || ilrType.trim.isEmpty) {
          warnInvalidSubtag("ILR", content, "type")
          "related"
        } else {
          ilrType
        }

        ilrRelationsTuples += ((synset, ilrName, content))
      }
      
      for ((tagName, content) <- synsetGenericRelationsTuples) {
        genericRelationsTuples += ((synset, tagName, content))
      }      
    }
  }  

  private def endGenericTag(tagName: String, content: String) {
    tagName.toLowerCase match {
      case "synonym" =>
        // skip
      case "word" =>
        // skip
      case "valency" =>
        //  skip
      case relName =>
        synsetGenericRelationsTuples += ((relName, content))
    }
  }
    
  override def endDocument {
    info("Synsets loaded") 
    createIlrRelations    
{'fraction_non_alphanumeric': 0.09598234807391744, 'fraction_numerical': 0.0, 'mean_word_length': 2.2774932208496534, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7314269', 'n_tokens_mistral': 3202, 'n_tokens_neox': 3112, 'n_words': 813}