<commit_before>package com.elsevier.soda

import scala.collection.JavaConversions.asScalaBuffer
import scala.collection.JavaConversions.asScalaIterator
import scala.collection.JavaConversions.collectionAsScalaIterable
import scala.collection.mutable.ArrayBuffer
import org.apache.commons.lang3.StringUtils
import org.apache.solr.client.solrj.SolrRequest
import org.apache.solr.client.solrj.impl.HttpSolrClient
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest
import org.apache.solr.common.SolrDocumentList
import org.apache.solr.common.params.CommonParams
import org.apache.solr.common.params.FacetParams
import org.apache.solr.common.params.ModifiableSolrParams
import org.apache.solr.common.util.ContentStreamBase
import org.apache.solr.common.util.NamedList
import org.springframework.stereotype.Service
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException
import java.util.Collections
import org.apache.solr.common.util.ContentStream
import org.apache.solr.common.SolrInputDocument
import java.util.regex.Pattern
import com.aliasi.chunk.RegExChunker

case class DictInfo(dictName: String, numEntries: Long)

@Service
class SodaService {

    val props = SodaUtils.props()
    val solrQueryUrl = props("SOLR_QUERY_URL")
    val solrUpdateUrls = props("SOLR_INDEX_URL").split(",").toList

    val querySolr = new HttpSolrClient(solrQueryUrl)
    val updateSolrs = solrUpdateUrls.map(url => new HttpSolrClient(url))
    
    val phraseChunker = new PhraseChunker()
    val stopwords = SodaUtils.stopwords()
    val sodaClient = new SodaClient()

    def annotate(text: String, lexiconName: String, 
            matchFlag: String): List[Annotation] = {
        val lexName = if (lexiconName.endsWith("-full")) 
                          lexiconName.substring(0, lexiconName.length() - 5)
                      else lexiconName
        val annotations = matchFlag match {
            case "exact" => tag(text, lexName, false)
            case "lower" => tag(text, lexName, true)
            case "punct" => chunkAndTag(text, lexName, "tagname_nrm")
            case "sort" => chunkAndTag(text, lexName, "tagname_srt")
            case "stem" => chunkAndTag(text, lexName, "tagname_stm")
            case _ => List()
        }
        if (lexiconName.endsWith("-full")) annotations
        else {
            annotations.filter(annotation => {
                val coveredText = annotation.props(AnnotationHelper.CoveredText)
                val isAbbrev = SodaUtils.isAbbreviation(coveredText)
                val isStopword = SodaUtils.isStopword(coveredText, stopwords)
                val isTooShort = SodaUtils.isTooShort(coveredText)
                isAbbrev || !(isStopword || isTooShort)
            })
        }
    }

    def tag(text: String, lexName: String, 
            lowerCaseInput: Boolean): List[Annotation] = {
        val params = new ModifiableSolrParams()
        params.add("overlaps", "LONGEST_DOMINANT_RIGHT")
        params.add("fq", buildFq(lexName, lowerCaseInput))
        params.add("fl", "id,tagtype,tagname_str")
        val req = new ContentStreamUpdateRequest("")
        val cstream = new ContentStreamBase.StringStream(
            if (lowerCaseInput) text.toLowerCase() else text)
        cstream.setContentType("text/plain")
        req.addContentStream(cstream)
        req.setMethod(SolrRequest.METHOD.POST)
        req.setPath("/tag")
        req.setParams(params)
        val resp = req.process(querySolr).getResponse()
        // extract the tags
        val tags = resp.get("tags").asInstanceOf[java.util.ArrayList[_]]
            .flatMap(tagInfo => {
                val tagList = tagInfo.asInstanceOf[NamedList[_]]
                val startOffset = tagList.get("startOffset").asInstanceOf[Int]
                val endOffset = tagList.get("endOffset").asInstanceOf[Int]
                val ids = tagList.get("ids").asInstanceOf[java.util.ArrayList[String]]
                ids.map(id => (id, startOffset, endOffset))
            }).toList
        // attach the confidences. If we are doing exact match, then we
        // can be lazy and attach confidence of 1 because these are exact
        // matches, otherwise we need to extract the tagname_str and calculate
        if (lowerCaseInput) {
            val idNameMap = resp.get("response").asInstanceOf[SolrDocumentList]
                .iterator()
                .map(doc => {
                    val id = doc.getFieldValue("id").asInstanceOf[String]
                    val tagType = doc.getFieldValues("tagtype")
                        .map(_.asInstanceOf[String])
                        .filter(_.equals(lexName))
                        .head
                    val names = doc.getFieldValues("tagname_str")
                        .map(_.asInstanceOf[String])
                        .toList
                    (id, (tagType, names))               
                }).toMap
            // remove trailing _ from id for lowercase records
            tags.map(tag => {
                val coveredText = text.substring(tag._2, tag._3)
                val conf = bestScore(coveredText, idNameMap(tag._1)._2) 
                Annotation("lx", tag._1.replace("_", ""), tag._2, tag._3,
                    Map(AnnotationHelper.CoveredText -> coveredText,
                        AnnotationHelper.Confidence -> AnnotationHelper.confToStr(conf),
                        AnnotationHelper.Lexicon -> idNameMap(tag._1)._1))
            })
        } else {
            val idTagtypeMap = resp.get("response")
                .asInstanceOf[SolrDocumentList]
                .iterator()
                .map(doc => {
                    val id = doc.getFieldValue("id").asInstanceOf[String]
                    val tagtype = doc.getFieldValues("tagtype")
                        .map(_.asInstanceOf[String])
                        .filter(_.equals(lexName))
                        .head
                    (id, tagtype)
                }).toMap
            tags.map(tag => {
                val coveredText = text.substring(tag._2, tag._3)
                Annotation("lx", tag._1, tag._2, tag._3, 
                    Map(AnnotationHelper.CoveredText -> coveredText, 
                        AnnotationHelper.Confidence -> "1.0",
                        AnnotationHelper.Lexicon -> idTagtypeMap(tag._1)))
                })
        }
    }
    
    def chunkAndTag(text: String, lexName: String, 
            matchOnField: String): List[Annotation] = {
        val phrases = phraseChunker.phraseChunk(text, "NP")
        val transformedPhrases = phrases.map(phrase => {
            val suffix = matchOnField.substring(matchOnField.lastIndexOf("_"))
            val transformedPhrase = suffix match {
                case "_nrm" => Normalizer.normalizeCasePunct(phrase._1)
                case "_srt" => Normalizer.sortWords(
                    Normalizer.normalizeCasePunct(phrase._1))
                case "_stm" => Normalizer.stemWords(
                    Normalizer.sortWords(
                    Normalizer.normalizeCasePunct(phrase._1)))
                  case _ => null
            }
            (transformedPhrase, phrase._2, phrase._3)
        })
        // run each of these phrases against
        val tags = ArrayBuffer[Annotation]()
        transformedPhrases.foreach(phrase => {<commit_msg>bugfix<commit_after>package com.elsevier.soda

import scala.collection.JavaConversions.asScalaBuffer
import scala.collection.JavaConversions.asScalaIterator
import scala.collection.JavaConversions.collectionAsScalaIterable
import scala.collection.mutable.ArrayBuffer
import org.apache.commons.lang3.StringUtils
import org.apache.solr.client.solrj.SolrRequest
import org.apache.solr.client.solrj.impl.HttpSolrClient
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest
import org.apache.solr.common.SolrDocumentList
import org.apache.solr.common.params.CommonParams
import org.apache.solr.common.params.FacetParams
import org.apache.solr.common.params.ModifiableSolrParams
import org.apache.solr.common.util.ContentStreamBase
import org.apache.solr.common.util.NamedList
import org.springframework.stereotype.Service
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException
import java.util.Collections
import org.apache.solr.common.util.ContentStream
import org.apache.solr.common.SolrInputDocument
import java.util.regex.Pattern
import com.aliasi.chunk.RegExChunker

case class DictInfo(dictName: String, numEntries: Long)

@Service
class SodaService {

    val props = SodaUtils.props()
    val solrQueryUrl = props("SOLR_QUERY_URL")
    val solrUpdateUrls = props("SOLR_INDEX_URL").split(",").toList

    val querySolr = new HttpSolrClient(solrQueryUrl)
    val updateSolrs = solrUpdateUrls.map(url => new HttpSolrClient(url))
    
    val phraseChunker = new PhraseChunker()
    val stopwords = SodaUtils.stopwords()
    val sodaClient = new SodaClient()

    def annotate(text: String, lexiconName: String, 
            matchFlag: String): List[Annotation] = {
        val lexName = if (lexiconName.endsWith("-full")) 
                          lexiconName.substring(0, lexiconName.length() - 5)
                      else lexiconName
        val annotations = matchFlag match {
            case "exact" => tag(text, lexName, false)
            case "lower" => tag(text, lexName, true)
            case "punct" => chunkAndTag(text, lexName, "tagname_nrm")
            case "sort" => chunkAndTag(text, lexName, "tagname_srt")
            case "stem" => chunkAndTag(text, lexName, "tagname_stm")
            case _ => List()
        }
        if (lexiconName.endsWith("-full")) annotations
        else {
            annotations.filter(annotation => {
                val coveredText = annotation.props(AnnotationHelper.CoveredText)
                val isAbbrev = SodaUtils.isAbbreviation(coveredText)
                val isStopword = SodaUtils.isStopword(coveredText, stopwords)
                val isTooShort = SodaUtils.isTooShort(coveredText)
                isAbbrev || !(isStopword || isTooShort)
            })
        }
    }

    def tag(text: String, lexName: String, 
            lowerCaseInput: Boolean): List[Annotation] = {
        val params = new ModifiableSolrParams()
        params.add("overlaps", "LONGEST_DOMINANT_RIGHT")
        params.add("fq", buildFq(lexName, lowerCaseInput))
        params.add("fl", "id,tagtype,tagname_str")
        val req = new ContentStreamUpdateRequest("")
        val cstream = new ContentStreamBase.StringStream(
            if (lowerCaseInput) text.toLowerCase() else text)
        cstream.setContentType("text/plain")
        req.addContentStream(cstream)
        req.setMethod(SolrRequest.METHOD.POST)
        req.setPath("/tag")
        req.setParams(params)
        val resp = req.process(querySolr).getResponse()
        // extract the tags
        val tags = resp.get("tags").asInstanceOf[java.util.ArrayList[_]]
            .flatMap(tagInfo => {
                val tagList = tagInfo.asInstanceOf[NamedList[_]]
                val startOffset = tagList.get("startOffset").asInstanceOf[Int]
                val endOffset = tagList.get("endOffset").asInstanceOf[Int]
                val ids = tagList.get("ids").asInstanceOf[java.util.ArrayList[String]]
                ids.map(id => (id, startOffset, endOffset))
            }).toList
        // attach the confidences. If we are doing exact match, then we
        // can be lazy and attach confidence of 1 because these are exact
        // matches, otherwise we need to extract the tagname_str and calculate
        if (lowerCaseInput) {
            val idNameMap = resp.get("response").asInstanceOf[SolrDocumentList]
                .iterator()
                .map(doc => {
                    val id = doc.getFieldValue("id").asInstanceOf[String]
                    val tagType = doc.getFieldValues("tagtype")
                        .map(_.asInstanceOf[String])
                        .filter(_.equals(lexName))
                        .head
                    val names = doc.getFieldValues("tagname_str")
                        .map(_.asInstanceOf[String])
                        .toList
                    (id, (tagType, names))               
                }).toMap
            // remove trailing _ from id for lowercase records
            tags.map(tag => {
                val coveredText = text.substring(tag._2, tag._3)
                val conf = bestScore(coveredText, idNameMap(tag._1)._2) 
                Annotation("lx", tag._1.replace("_", ""), tag._2, tag._3,
                    Map(AnnotationHelper.CoveredText -> coveredText,
                        AnnotationHelper.Confidence -> AnnotationHelper.confToStr(conf),
                        AnnotationHelper.Lexicon -> idNameMap(tag._1)._1))
            })
        } else {
            val idTagtypeMap = resp.get("response")
                .asInstanceOf[SolrDocumentList]
                .iterator()
                .map(doc => {
                    val id = doc.getFieldValue("id").asInstanceOf[String]
                    val tagtype = doc.getFieldValues("tagtype")
                        .map(_.asInstanceOf[String])
                        .filter(_.equals(lexName))
                        .head
                    (id, tagtype)
                }).toMap
            tags.map(tag => {
                val coveredText = text.substring(tag._2, tag._3)
                Annotation("lx", tag._1, tag._2, tag._3, 
                    Map(AnnotationHelper.CoveredText -> coveredText, 
                        AnnotationHelper.Confidence -> "1.0",
                        AnnotationHelper.Lexicon -> lexName))
//                        AnnotationHelper.Lexicon -> idTagtypeMap(tag._1)))
                })
        }
    }
    
    def chunkAndTag(text: String, lexName: String, 
            matchOnField: String): List[Annotation] = {
        val phrases = phraseChunker.phraseChunk(text, "NP")
        val transformedPhrases = phrases.map(phrase => {
            val suffix = matchOnField.substring(matchOnField.lastIndexOf("_"))
            val transformedPhrase = suffix match {
                case "_nrm" => Normalizer.normalizeCasePunct(phrase._1)
                case "_srt" => Normalizer.sortWords(
                    Normalizer.normalizeCasePunct(phrase._1))
                case "_stm" => Normalizer.stemWords(
                    Normalizer.sortWords(
                    Normalizer.normalizeCasePunct(phrase._1)))
                  case _ => null
            }
            (transformedPhrase, phrase._2, phrase._3)
        })
        // run each of these phrases against
        val tags = ArrayBuffer[Annotation]()
        transformedPhrases.foreach(phrase => {
{'fraction_non_alphanumeric': 0.10299003322259136, 'fraction_numerical': 0.003525662756797071, 'mean_word_length': 2.0935402684563758, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4549356', 'n_tokens_mistral': 4228, 'n_tokens_neox': 3991, 'n_words': 766}