<commit_before>/*
 * (C) 2010-2012 ICM UW. All rights reserved.
 */

package pl.edu.icm.coansys.citations.mappers

import org.apache.hadoop.io.{Text, BytesWritable}
import org.apache.hadoop.mapreduce.Mapper
import pl.edu.icm.cermine.bibref.{CRFBibReferenceParser, BibReferenceParser}
import pl.edu.icm.cermine.bibref.model.BibEntry
import pl.edu.icm.coansys.citations.data.CitationEntity
import pl.edu.icm.coansys.citations.indices.AuthorIndex
import pl.edu.icm.coansys.citations.jobs.Matcher
import pl.edu.icm.coansys.importers.models.DocumentProtos.ReferenceMetadata


class HeuristicAdder extends Mapper[BytesWritable, BytesWritable, BytesWritable, Text] {
  type Context = Mapper[BytesWritable, BytesWritable, BytesWritable, Text]#Context
  val keyWritable = new BytesWritable()
  val valueWritable = new Text()
  var parser: BibReferenceParser[BibEntry] = null
  var index: AuthorIndex = null

  override def setup(context: Context) {
    val parserModelUri = context.getConfiguration.get("bibref.parser.model")
    parser = new CRFBibReferenceParser(parserModelUri)

    val authorIndexUri = context.getConfiguration.get("index.author")
    index = new AuthorIndex(authorIndexUri)
  }

  override def map(key: BytesWritable, value: BytesWritable, context: Context) {
    val meta = ReferenceMetadata.parseFrom(key.copyBytes())
    val cit = CitationEntity.fromUnparsedReferenceMetadata(parser, meta)
    val bytes = cit.toTypedBytes
    keyWritable.set(bytes, 0, bytes.length)
    Matcher.approximatelyMatchingDocuments(cit, index).foreach {
      case (entityId) =>
        valueWritable.set(entityId)
        context.write(keyWritable, valueWritable)
    }
  }

  override def cleanup(context: Context) {
    if (index != null)
      index.close()
  }<commit_msg>enable no parsing<commit_after>/*
 * (C) 2010-2012 ICM UW. All rights reserved.
 */

package pl.edu.icm.coansys.citations.mappers

import org.apache.hadoop.io.{Text, BytesWritable}
import org.apache.hadoop.mapreduce.Mapper
import pl.edu.icm.cermine.bibref.{CRFBibReferenceParser, BibReferenceParser}
import pl.edu.icm.cermine.bibref.model.BibEntry
import pl.edu.icm.coansys.citations.data.CitationEntity
import pl.edu.icm.coansys.citations.indices.AuthorIndex
import pl.edu.icm.coansys.citations.jobs.Matcher
import pl.edu.icm.coansys.importers.models.DocumentProtos.ReferenceMetadata


class HeuristicAdder extends Mapper[BytesWritable, BytesWritable, BytesWritable, Text] {
  type Context = Mapper[BytesWritable, BytesWritable, BytesWritable, Text]#Context
  val keyWritable = new BytesWritable()
  val valueWritable = new Text()
  var parser: BibReferenceParser[BibEntry] = null
  var index: AuthorIndex = null

  override def setup(context: Context) {
    val parserModelUri = context.getConfiguration.get("bibref.parser.model")
    if (!parserModelUri.isEmpty)
      parser = new CRFBibReferenceParser(parserModelUri)

    val authorIndexUri = context.getConfiguration.get("index.author")
    index = new AuthorIndex(authorIndexUri)
  }

  override def map(key: BytesWritable, value: BytesWritable, context: Context) {
    val meta = ReferenceMetadata.parseFrom(key.copyBytes())
    val cit =
      if (parser == null) {
        CitationEntity.fromUnparsedReferenceMetadata(parser, meta)
      } else {
        CitationEntity.fromReferenceMetadata(meta)
      }
    val bytes = cit.toTypedBytes
    keyWritable.set(bytes, 0, bytes.length)
    Matcher.approximatelyMatchingDocuments(cit, index).foreach {
      case (entityId) =>
        valueWritable.set(entityId)
        context.write(keyWritable, valueWritable)
    }
  }

  override def cleanup(context: Context) {
    if (index != null)
      index.close()
  }
{'fraction_non_alphanumeric': 0.10159848279599025, 'fraction_numerical': 0.004876727174207532, 'mean_word_length': 4.46962962962963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3406590', 'n_tokens_mistral': 1209, 'n_tokens_neox': 1144, 'n_words': 227}