<commit_before>package code.model

import _root_.net.liftweb.mapper._
import _root_.net.liftweb.util._
import _root_.net.liftweb.common._
import _root_.scala.xml.{NodeSeq, Text}
import org.apache.lucene.document.{Document,Field}
import com.sun.mail.smtp.SMTPMessage

import java.io.{IOException, InputStream, ByteArrayInputStream}


class Message extends LongKeyedMapper[Message] with IdPK {
  def getSingleton = Message
  object msgBody extends MappedBinary(this)
  object subject extends MappedText(this)
  object sender extends MappedEmail(this, 256)
  object sentDate extends MappedDateTime(this)
  object messageId extends MappedString(this, 256)
  object textContent extends MappedText(this)
  object lazy_recipients extends HasManyThrough(this, Recipient,
    MessageRecipient, MessageRecipient.recipient, MessageRecipient.message)

  def recipients = MessageRecipient.findAll(By(MessageRecipient.message, this.id)).map(_.recipient.obj.open_!)

  def recipientsPrintable() = {
    val rlinks = recipients.map({x => val el = <a href={"/recipient/" + x.primaryKeyField}>{x.addressIndex}</a>
                                      el.asInstanceOf[NodeSeq]
                                })
    rlinks.reduceLeft((x,y) => x ++ Text(", ") ++ y)
  }

  def getHeaders() : String = {
    val msg = new SMTPMessage(null, new ByteArrayInputStream(msgBody))
    val headers = msg.getAllHeaderLines()
    val headertext = new StringBuilder()
    while(headers.hasMoreElements()) {
      headertext.append(headers.nextElement()).append("\n")
    }
    headertext.toString()
  }

  // Convert to a Lucene Document for indexing
  def toDocument(): Document = {
    val doc = new Document
    doc.add(mkField(id,id.toString,true,false))
    doc.add(mkField(textContent,textContent,true,true))
    doc.add(mkField(subject,subject,true,true))
    doc.add(mkField(sender,sender,true,true))
    def mkField[A,B <: net.liftweb.mapper.Mapper[B]](
        name: MappedField[A,B],
        value: String,
        store: Boolean,
        searchable: Boolean): Field = {
      val fieldStore = if (store) Field.Store.YES else Field.Store.NO
      val indexed = if (searchable) Field.Index.ANALYZED else Field.Index.NO
      new Field(name.dbColumnName,value,fieldStore,indexed)
    }
    doc
  }

}
<commit_msg>Prevent null values in Fields.<commit_after>package code.model

import _root_.net.liftweb.mapper._
import _root_.net.liftweb.util._
import _root_.net.liftweb.common._
import _root_.scala.xml.{NodeSeq, Text}
import org.apache.lucene.document.{Document,Field}
import com.sun.mail.smtp.SMTPMessage

import java.io.{IOException, InputStream, ByteArrayInputStream}


class Message extends LongKeyedMapper[Message] with IdPK {
  def getSingleton = Message
  object msgBody extends MappedBinary(this)
  object subject extends MappedText(this)
  object sender extends MappedEmail(this, 256)
  object sentDate extends MappedDateTime(this)
  object messageId extends MappedString(this, 256)
  object textContent extends MappedText(this)
  object lazy_recipients extends HasManyThrough(this, Recipient,
    MessageRecipient, MessageRecipient.recipient, MessageRecipient.message)

  def recipients = MessageRecipient.findAll(By(MessageRecipient.message, this.id)).map(_.recipient.obj.open_!)

  def recipientsPrintable() = {
    val rlinks = recipients.map({x => val el = <a href={"/recipient/" + x.primaryKeyField}>{x.addressIndex}</a>
                                      el.asInstanceOf[NodeSeq]
                                })
    rlinks.reduceLeft((x,y) => x ++ Text(", ") ++ y)
  }

  def getHeaders() : String = {
    val msg = new SMTPMessage(null, new ByteArrayInputStream(msgBody))
    val headers = msg.getAllHeaderLines()
    val headertext = new StringBuilder()
    while(headers.hasMoreElements()) {
      headertext.append(headers.nextElement()).append("\n")
    }
    headertext.toString()
  }

  // Convert to a Lucene Document for indexing
  def toDocument(): Document = {
    val doc = new Document
    doc.add(mkField(id,id.toString,true,false))
    doc.add(mkField(textContent,textContent,true,true))
    doc.add(mkField(subject,subject,true,true))
    doc.add(mkField(sender,sender,true,true))
    def mkField[A,B <: net.liftweb.mapper.Mapper[B]](
        name: MappedField[A,B],
        value: String,
        store: Boolean,
        searchable: Boolean): Field = {
      val fieldStore = if (store) Field.Store.YES else Field.Store.NO
      val indexed = if (searchable) Field.Index.ANALYZED else Field.Index.NO
      val valNoNull = if (value==null) "" else value
      new Field(name.dbColumnName,valNoNull,fieldStore,indexed)
    }
    doc
  }

}

{'fraction_non_alphanumeric': 0.11421210820094461, 'fraction_numerical': 0.0025762129669386004, 'mean_word_length': 3.687122736418511, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5436937', 'n_tokens_mistral': 1489, 'n_tokens_neox': 1415, 'n_words': 304}