<commit_before>package edu.umass.cs.iesl.bibmogrify.compare

import edu.umass.cs.iesl.bibmogrify.model.RichStructuredCitation._
import edu.umass.cs.iesl.bibmogrify.pipeline.Transformer
import edu.umass.cs.iesl.bibmogrify.NamedPlugin
import edu.umass.cs.iesl.scalacommons.NonemptyString
import edu.umass.cs.iesl.scalacommons.StringUtils._
import edu.umass.cs.iesl.bibmogrify.model.{Person, StructuredCitation}

/**
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 * @version $Id$
 */
class StringZipTransformer(trans: Transformer[StructuredCitation, NonemptyString] with NamedPlugin)
		extends Transformer[StructuredCitation, (NonemptyString, StructuredCitation)] //with NamedPlugin
	{
	//val name = "zip("+trans.name+")"
	def apply(cm: StructuredCitation) = trans(cm).toSeq.map(((_, cm)))
	}

/**
 * render the title in a normalized form using only characters from the IUPAC amino acid alphabet, as a hack to allow comparing titles with usearch.
 */
object AminoAcidTitleHash extends Transformer[StructuredCitation, NonemptyString] with NamedPlugin
	{
	val name = "AAHash"

	// could also remove stopwords, etc.
	// AA codes are all uppercase letters except J, O, and U.
	// but we want to consider those.
	// therefore map WXYZ -> Z, J->W, O->X, U->Y
	//private def aaize(s: String): String = s.toUpperCase.deAccent.removeWhitespace.replaceAll("[^A-Z]", "").replaceAll("WXY", "Z").replaceAll("J", "W")
	//                                       .replaceAll("O", "X").replaceAll("U", "Y")
	// by removing vowels, O and U are gone anyway, so we can just map J to A and be done with tit
	private def aaize(s: String): String = s.toUpperCase.deAccent.removeWhitespace.replaceAll("[^A-Z]", "").removeVowels.replaceAll("J", "A")

	//private def limit(s:String, len: Int) = s.substring(0,math.max(s.length, len)-1)
	def apply(cm: StructuredCitation) =
		{
		// implicit emptyStringToNone didn't work right; just be verbose
		// waiting for robust last name extraction
		val result: Option[NonemptyString] = emptyStringToNone(aaize(cm.title.getOrElse("[ERROR: EMPTY TITLE]") + cm.firstAuthorLastName))
		result
		}
	}

// do manually for now; would be nice to automate
object ZipAminoAcidTitleHash
		extends StringZipTransformer(AminoAcidTitleHash) with Transformer[StructuredCitation, (NonemptyString, StructuredCitation)] with NamedPlugin
	{
	val name = "zipAAHash"
	}





class FUSEPaperCorefBundle(val entityId:String, val mentionId:String, val citation:StructuredCitation)
/**
 * render the title in a normalized form using only characters from the IUPAC amino acid alphabet, as a hack to allow comparing titles with usearch.
 */
object FUSEPaperCorefHash extends Transformer[StructuredCitation, NonemptyString] with NamedPlugin{
	val name = "FPCHash"
	def apply(cm: StructuredCitation) ={
    if(cm.authors.size>0){
      //println("last name: "+cm.authors.head.agent.longestSurName)

      cm.authors.head.agent match{
        case a:Person =>{
          /*
          if(a.name!=None)println("NAME: "+a.name.get.longestSurName) else println("Name is none.")
          println("Email: "+a.email)
          println("Phone: "+a.phone)
          println("Affiliations: "+a.affiliations)
          println("Homepages: "+a.homepages)
          println("Identifiers: "+a.identifiers)
          */
          /*
	val address     : Option[Address]                   = None
	val email       : Option[NonemptyString]            = None
	val phone       : Option[NonemptyString]            = None
	val affiliations: Seq[Institution]                  = Nil
	val homepages   : Seq[URL]                          = Nil
	val identifiers : Seq[PersonIdentifier]             = Nil

           */
        }
        case _ => throw new Exception("WHy isn't htis an author? "+cm.authors.head.agent.getClass())
      }

    }
		// could also remove stopwords, etc.
		// AA codes are all uppercase letters except J, O, and U.
		// but we want to consider those.
		// therefore map WXYZ -> Z, J->W, O->X, U->Y
		def aaize(s: String): String ={
      val result =
      s.removeAllButWord.replaceAll(" ","")
        .toUpperCase.replaceAll("WXY", "Z")
        .replaceAll("J", "W").replaceAll("O","X")
        .replaceAll("U","Y")
        result.substring(0,math.min(result.length-1,20))  }
 //def limit(s:String, len: Int) = s.substring(0,math.max(s.length, len)-1)
		val result: Option[NonemptyString] = emptyStringToNone(aaize(cm.title.getOrElse("[ERROR: EMPTY TITLE]")+ cm.firstAuthorLastName))
		result
		}
	}
// do manually for now; would be nice to automate
object FUSEZipPaperCorefHash
  extends StringZipTransformer(FUSEPaperCorefHash)<commit_msg>more features, removed aa hash features<commit_after>package edu.umass.cs.iesl.bibmogrify.compare

import edu.umass.cs.iesl.bibmogrify.model.RichStructuredCitation._
import edu.umass.cs.iesl.bibmogrify.pipeline.Transformer
import edu.umass.cs.iesl.bibmogrify.NamedPlugin
import edu.umass.cs.iesl.scalacommons.NonemptyString
import edu.umass.cs.iesl.scalacommons.StringUtils._
import edu.umass.cs.iesl.bibmogrify.model.{Person, StructuredCitation}

/**
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 * @version $Id$
 */
class StringZipTransformer(trans: Transformer[StructuredCitation, NonemptyString] with NamedPlugin)
		extends Transformer[StructuredCitation, (NonemptyString, StructuredCitation)] //with NamedPlugin
	{
	//val name = "zip("+trans.name+")"
	def apply(cm: StructuredCitation) = trans(cm).toSeq.map(((_, cm)))
	}

/**
 * render the title in a normalized form using only characters from the IUPAC amino acid alphabet, as a hack to allow comparing titles with usearch.
 */
object AminoAcidTitleHash extends Transformer[StructuredCitation, NonemptyString] with NamedPlugin
	{
	val name = "AAHash"

	// could also remove stopwords, etc.
	// AA codes are all uppercase letters except J, O, and U.
	// but we want to consider those.
	// therefore map WXYZ -> Z, J->W, O->X, U->Y
	//private def aaize(s: String): String = s.toUpperCase.deAccent.removeWhitespace.replaceAll("[^A-Z]", "").replaceAll("WXY", "Z").replaceAll("J", "W")
	//                                       .replaceAll("O", "X").replaceAll("U", "Y")
	// by removing vowels, O and U are gone anyway, so we can just map J to A and be done with tit
	private def aaize(s: String): String = s.toUpperCase.deAccent.removeWhitespace.replaceAll("[^A-Z]", "").removeVowels.replaceAll("J", "A")

	//private def limit(s:String, len: Int) = s.substring(0,math.max(s.length, len)-1)
	def apply(cm: StructuredCitation) =
		{
		// implicit emptyStringToNone didn't work right; just be verbose
		// waiting for robust last name extraction
		val result: Option[NonemptyString] = emptyStringToNone(aaize(cm.title.getOrElse("[ERROR: EMPTY TITLE]") + cm.firstAuthorLastName))
		result
		}
	}

// do manually for now; would be nice to automate
object ZipAminoAcidTitleHash
		extends StringZipTransformer(AminoAcidTitleHash) with Transformer[StructuredCitation, (NonemptyString, StructuredCitation)] with NamedPlugin
	{
	val name = "zipAAHash"
	}





class FUSEPaperCorefBundle(val entityId:String, val mentionId:String, val citation:StructuredCitation)
/**
 * render the title in a normalized form using only characters from the IUPAC amino acid alphabet, as a hack to allow comparing titles with usearch.
 */
object FUSEPaperCorefHash extends Transformer[StructuredCitation, NonemptyString] with NamedPlugin{
	val name = "FPCHash"
	def apply(cm: StructuredCitation) ={
    def limit(s:String,len:Int):String = s.substring(0,math.min(s.length-1,len))
		def project(s: String): String ={
      limit(
        s.removeAllButWord.replaceAll(" ","")
          .toUpperCase
          .deAccent
          .replaceAll("[^A-Z]","")
          .removeVowels,20
      )
    }
    //	private def aaize(s: String): String = s.toUpperCase.deAccent.removeWhitespace.replaceAll("[^A-Z]", "").removeVowels.replaceAll("J", "A")
 //def limit(s:String, len: Int) = s.substring(0,math.max(s.length, len)-1)
		val result: Option[NonemptyString] = emptyStringToNone(project(cm.title.getOrElse("[ERROR: EMPTY TITLE]")+ cm.firstAuthorLastName))
		result
		}
	}
// do manually for now; would be nice to automate
object FUSEZipPaperCorefHash
  extends StringZipTransformer(FUSEPaperCorefHash)
{'fraction_non_alphanumeric': 0.1280465623863223, 'fraction_numerical': 0.0020613556444767796, 'mean_word_length': 3.8037274315666862, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3637284', 'n_tokens_mistral': 2818, 'n_tokens_neox': 2663, 'n_words': 668}