<commit_before>package pl.edu.icm.coansys.citations.mappers

import collection.JavaConversions._
import org.apache.hadoop.io.{BytesWritable, Text, Writable}
import org.apache.hadoop.mapreduce.Mapper
import pl.edu.icm.cermine.bibref.CRFBibReferenceParser
import pl.edu.icm.coansys.citations.data.MatchableEntity
import pl.edu.icm.coansys.models.DocumentProtos.DocumentWrapper
import org.slf4j.LoggerFactory

/**
 * @author <NAME> (<EMAIL>)
 */
class ReferenceExtractorAndParser extends Mapper[Writable, BytesWritable, Text, BytesWritable] {
  type Context = Mapper[Writable, BytesWritable, Text, BytesWritable]#Context
  val maxSupportedCitationLength = 2000
  private val logger = LoggerFactory.getLogger(classOf[ReferenceExtractorAndParser])
  private val keyWritable = new Text()
  private val valueWritable = new BytesWritable()
  private var parser: CRFBibReferenceParser = null

  override def setup(context: Context) {
    val model = context.getConfiguration.get("crf.reference.parser.model")
    parser =
      if(model != null)
        new CRFBibReferenceParser(model)
      else
        new CRFBibReferenceParser(
          this.getClass.getResourceAsStream("/pl/edu/icm/cermine/bibref/acrf-small.ser.gz"))
  }

  override def map(key: Writable, value: BytesWritable, context: Context) {
    val wrapper = DocumentWrapper.parseFrom(value.copyBytes())
    wrapper.getDocumentMetadata.getReferenceList.filterNot{_.getRawCitationText.isEmpty}.foreach {
      case ref =>
        if (ref.getRawCitationText.length > maxSupportedCitationLength) {
          logger.warn(s"Citation ${ref.getPosition} in document ${wrapper.getRowId} exceeds max supported citation length. Omitted.")
        } else {
          val entity = MatchableEntity.fromUnparsedReferenceMetadata(parser, ref)
          val bytes = entity.data.toByteArray
          keyWritable.set(entity.id)
          valueWritable.set(bytes, 0, bytes.length)
          context.write(keyWritable, valueWritable)
        }
    }
  }
}<commit_msg>take a correct doc id<commit_after>package pl.edu.icm.coansys.citations.mappers

import collection.JavaConversions._
import org.apache.hadoop.io.{BytesWritable, Text, Writable}
import org.apache.hadoop.mapreduce.Mapper
import pl.edu.icm.cermine.bibref.CRFBibReferenceParser
import pl.edu.icm.coansys.citations.data.MatchableEntity
import pl.edu.icm.coansys.models.DocumentProtos.DocumentWrapper
import org.slf4j.LoggerFactory
import pl.edu.icm.coansys.citations.data.entity_id.CitEntityId

/**
 * @author <NAME> (<EMAIL>)
 */
class ReferenceExtractorAndParser extends Mapper[Writable, BytesWritable, Text, BytesWritable] {
  type Context = Mapper[Writable, BytesWritable, Text, BytesWritable]#Context
  val maxSupportedCitationLength = 2000
  private val logger = LoggerFactory.getLogger(classOf[ReferenceExtractorAndParser])
  private val keyWritable = new Text()
  private val valueWritable = new BytesWritable()
  private var parser: CRFBibReferenceParser = null

  override def setup(context: Context) {
    val model = context.getConfiguration.get("crf.reference.parser.model")
    parser =
      if(model != null)
        new CRFBibReferenceParser(model)
      else
        new CRFBibReferenceParser(
          this.getClass.getResourceAsStream("/pl/edu/icm/cermine/bibref/acrf-small.ser.gz"))
  }

  override def map(key: Writable, value: BytesWritable, context: Context) {
    val wrapper = DocumentWrapper.parseFrom(value.copyBytes())
    wrapper.getDocumentMetadata.getReferenceList.filterNot{_.getRawCitationText.isEmpty}.foreach {
      case ref =>
        if (ref.getRawCitationText.length > maxSupportedCitationLength) {
          logger.warn(s"Citation ${ref.getPosition} in document ${wrapper.getRowId} exceeds max supported citation length. Omitted.")
        } else {
          val citId = new CitEntityId(wrapper.getDocumentMetadata.getKey, ref.getPosition)
          val entity = MatchableEntity.fromUnparsedReference(parser, citId.toString, ref.getRawCitationText)
          val bytes = entity.data.toByteArray
          keyWritable.set(entity.id)
          valueWritable.set(bytes, 0, bytes.length)
          context.write(keyWritable, valueWritable)
        }
    }
  }
}
{'fraction_non_alphanumeric': 0.09740414384377233, 'fraction_numerical': 0.0028578232912598238, 'mean_word_length': 4.121951219512195, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5665744', 'n_tokens_mistral': 1275, 'n_tokens_neox': 1237, 'n_words': 240}