<commit_before>package ru.org.linux.search

import com.typesafe.scalalogging.slf4j.StrictLogging
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.aggregations.Aggregations
import org.elasticsearch.search.aggregations.bucket.filter.Filter
import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms
import org.elasticsearch.search.aggregations.bucket.terms.Terms
import org.joda.time.DateTime
import org.joda.time.format.ISODateTimeFormat
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import org.springframework.web.util.UriComponentsBuilder
import ru.org.linux.group.GroupDao
import ru.org.linux.section.SectionService
import ru.org.linux.tag.{TagRef, TagService}
import ru.org.linux.user.{User, UserDao}
import ru.org.linux.util.StringUtil
import ru.org.linux.util.URLUtil._

import scala.beans.BeanProperty
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._

case class SearchItem (
  @BeanProperty title:String,
  @BeanProperty postdate:DateTime,
  @BeanProperty user:User, // TODO use UserRef
  @BeanProperty message:String,
  @BeanProperty url:String,
  @BeanProperty score:Float,
  @BeanProperty comment:Boolean,
  @BeanProperty tags:java.util.List[TagRef]
)

@Service
class SearchResultsService @Autowired() (
  userDao:UserDao, sectionService:SectionService, groupDao:GroupDao
) extends StrictLogging {
  import ru.org.linux.search.SearchResultsService._

  def prepareAll(docs:java.lang.Iterable[SearchHit]) = (docs map prepare).asJavaCollection

  def prepare(doc:SearchHit):SearchItem = {
    val author = userDao.getUser(doc.getFields.get("author").getValue[String])

    val postdate = isoDateTime.parseDateTime(doc.getFields.get("postdate").getValue[String])

    val comment = doc.getFields.get("is_comment").getValue[Boolean]

    val tags = if (comment) {
      Seq()
    } else {
      if (doc.getFields.containsKey("tag")) {
        doc.getFields.get("tag").getValues.map(
          tag => TagService.tagRef(tag.toString))
      } else {
        Seq()
      }
    }

    SearchItem(
      title = getTitle(doc),
      postdate = postdate,
      user = author,
      url = getUrl(doc),
      score = doc.getScore,
      comment = comment,
      message = getMessage(doc),
      tags = tags
    )
  }

  private def getTitle(doc: SearchHit):String = {
    val itemTitle = if (doc.getHighlightFields.containsKey("title")) {
      Some(doc.getHighlightFields.get("title").fragments()(0).string)
    } else if (doc.getFields.containsKey("title")) {
      Some(StringUtil.escapeHtml(doc.getFields.get("title").getValue[String]))
    } else {
      None
    }

    val topicTitle = if (doc.getHighlightFields.containsKey("topic_title")) {
      doc.getHighlightFields.get("topic_title").fragments()(0).string
    } else {
      StringUtil.escapeHtml(doc.getFields.get("topic_title").getValue[String])
    }

    itemTitle.filter(!_.trim.isEmpty).getOrElse(topicTitle)
  }

  private def getMessage(doc: SearchHit): String = {
    if (doc.getHighlightFields.containsKey("message")) {
      doc.getHighlightFields.get("message").fragments()(0).string
    } else {
      val fullMessage = doc.getFields.get("message").getValue[String]
      if (fullMessage.length > SearchViewer.MESSAGE_FRAGMENT) {
        fullMessage.substring(0, SearchViewer.MESSAGE_FRAGMENT)
      } else {
        fullMessage
      }
    }
  }<commit_msg>поправил экранирование HTML в поиске<commit_after>package ru.org.linux.search

import com.typesafe.scalalogging.slf4j.StrictLogging
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.aggregations.Aggregations
import org.elasticsearch.search.aggregations.bucket.filter.Filter
import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms
import org.elasticsearch.search.aggregations.bucket.terms.Terms
import org.joda.time.DateTime
import org.joda.time.format.ISODateTimeFormat
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import org.springframework.web.util.UriComponentsBuilder
import ru.org.linux.group.GroupDao
import ru.org.linux.section.SectionService
import ru.org.linux.tag.{TagRef, TagService}
import ru.org.linux.user.{User, UserDao}
import ru.org.linux.util.StringUtil
import ru.org.linux.util.URLUtil._

import scala.beans.BeanProperty
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._

case class SearchItem (
  @BeanProperty title:String,
  @BeanProperty postdate:DateTime,
  @BeanProperty user:User, // TODO use UserRef
  @BeanProperty message:String,
  @BeanProperty url:String,
  @BeanProperty score:Float,
  @BeanProperty comment:Boolean,
  @BeanProperty tags:java.util.List[TagRef]
)

@Service
class SearchResultsService @Autowired() (
  userDao:UserDao, sectionService:SectionService, groupDao:GroupDao
) extends StrictLogging {
  import ru.org.linux.search.SearchResultsService._

  def prepareAll(docs:java.lang.Iterable[SearchHit]) = (docs map prepare).asJavaCollection

  def prepare(doc:SearchHit):SearchItem = {
    val author = userDao.getUser(doc.getFields.get("author").getValue[String])

    val postdate = isoDateTime.parseDateTime(doc.getFields.get("postdate").getValue[String])

    val comment = doc.getFields.get("is_comment").getValue[Boolean]

    val tags = if (comment) {
      Seq()
    } else {
      if (doc.getFields.containsKey("tag")) {
        doc.getFields.get("tag").getValues.map(
          tag => TagService.tagRef(tag.toString))
      } else {
        Seq()
      }
    }

    SearchItem(
      title = getTitle(doc),
      postdate = postdate,
      user = author,
      url = getUrl(doc),
      score = doc.getScore,
      comment = comment,
      message = getMessage(doc),
      tags = tags
    )
  }

  private def getTitle(doc: SearchHit):String = {
    val itemTitle = if (doc.getHighlightFields.containsKey("title")) {
      Some(doc.getHighlightFields.get("title").fragments()(0).string)
    } else if (doc.getFields.containsKey("title")) {
      Some(StringUtil.escapeHtml(doc.getFields.get("title").getValue[String]))
    } else {
      None
    }

    val topicTitle = if (doc.getHighlightFields.containsKey("topic_title")) {
      doc.getHighlightFields.get("topic_title").fragments()(0).string
    } else {
      StringUtil.escapeHtml(doc.getFields.get("topic_title").getValue[String])
    }

    itemTitle.filter(!_.trim.isEmpty).getOrElse(topicTitle)
  }

  private def getMessage(doc: SearchHit): String = {
    if (doc.getHighlightFields.containsKey("message")) {
      doc.getHighlightFields.get("message").fragments()(0).string
    } else {
      val rawMessage = doc.getFields.get("message").getValue[String].take(SearchViewer.MESSAGE_FRAGMENT)

      StringUtil.escapeHtml(rawMessage)
    }
  }
{'fraction_non_alphanumeric': 0.12020348837209302, 'fraction_numerical': 0.001308139534883721, 'mean_word_length': 4.212878787878788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2890457', 'n_tokens_mistral': 2118, 'n_tokens_neox': 2094, 'n_words': 328}