<commit_before>package edu.umass.ciir.kbbridge.nlp

import collection.mutable.ListBuffer
import edu.umass.ciir.kbbridge.nlp.NlpData.{Token, NlpXmlNerMention}
import edu.umass.ciir.models.StopWordList

/**
 *   //==================================
  //== Fall back for when NLP data is not available ==
  //--
  //-

 */
object PoorMansNlpExtractor {

  def textToTokenWithOffset(cleanText:String): ListBuffer[NlpData.Token] = {
    val tokens = new ListBuffer[Token]
    var idxText = cleanText.toCharArray.zipWithIndex
    while (idxText != null && !idxText.isEmpty) {
      val (chunk, rest) = idxText.span(_._1.isLetterOrDigit)
      idxText = rest.dropWhile(!_._1.isLetterOrDigit)

      if (chunk.length > 1) {
        val rawWord = chunk.map(_._1).mkString
        val beginOffset = chunk.head._2
        val endOffset = chunk.last._2 + 1
        tokens += Token(rawWord, "UNK", "?", rawWord, beginOffset, endOffset)
      }
    }
    tokens
  }


  def computeSentenceOffsets(cleanText:String):Seq[(Int,Int)] = {
    var idxText = cleanText.toCharArray.zipWithIndex


    // find sentence boundaries
    val sentOffsetToSentId0 = idxText.filter(_._1=='.').map(_._2).zipWithIndex
    val sentOffsetToSentId = sentOffsetToSentId0.map(offsetId => (offsetId._1, offsetId._2+1)) // sentence ids start with 1
    sentOffsetToSentId
  }
  def offsetToSentId(offset:Int, sentOffsetToSentId:Seq[(Int,Int)]){
    val lastOpt = sentOffsetToSentId.takeWhile(offsetId => offsetId._1 <= offset).lastOption     // "." included therefore <=
    lastOpt match {
      case Some((sentEndOffset, sentId)) => {
        if (offset > sentEndOffset) sentId +1  // in case last sentence did not end on a "."
        else sentId
      }<commit_msg>Fixed return type<commit_after>package edu.umass.ciir.kbbridge.nlp

import collection.mutable.ListBuffer
import edu.umass.ciir.kbbridge.nlp.NlpData.{Token, NlpXmlNerMention}
import edu.umass.ciir.models.StopWordList

/**
 *   //==================================
  //== Fall back for when NLP data is not available ==
  //--
  //-

 */
object PoorMansNlpExtractor {

  def textToTokenWithOffset(cleanText:String): ListBuffer[NlpData.Token] = {
    val tokens = new ListBuffer[Token]
    var idxText = cleanText.toCharArray.zipWithIndex
    while (idxText != null && !idxText.isEmpty) {
      val (chunk, rest) = idxText.span(_._1.isLetterOrDigit)
      idxText = rest.dropWhile(!_._1.isLetterOrDigit)

      if (chunk.length > 1) {
        val rawWord = chunk.map(_._1).mkString
        val beginOffset = chunk.head._2
        val endOffset = chunk.last._2 + 1
        tokens += Token(rawWord, "UNK", "?", rawWord, beginOffset, endOffset)
      }
    }
    tokens
  }


  def computeSentenceOffsets(cleanText:String):Seq[(Int,Int)] = {
    var idxText = cleanText.toCharArray.zipWithIndex


    // find sentence boundaries
    val sentOffsetToSentId0 = idxText.filter(_._1=='.').map(_._2).zipWithIndex
    val sentOffsetToSentId = sentOffsetToSentId0.map(offsetId => (offsetId._1, offsetId._2+1)) // sentence ids start with 1
    sentOffsetToSentId
  }
  def offsetToSentId(offset:Int, sentOffsetToSentId:Seq[(Int,Int)]):Int = {
    val lastOpt = sentOffsetToSentId.takeWhile(offsetId => offsetId._1 <= offset).lastOption     // "." included therefore <=
    lastOpt match {
      case Some((sentEndOffset, sentId)) => {
        if (offset > sentEndOffset) sentId + 1  // in case last sentence did not end on a "."
        else sentId
      }
{'fraction_non_alphanumeric': 0.14228110599078342, 'fraction_numerical': 0.009792626728110598, 'mean_word_length': 3.3089330024813894, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3873533', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1114, 'n_words': 228}