<commit_before>package pl.edu.icm.coansys.citations

import com.nicta.scoobi.application.ScoobiApp
import com.nicta.scoobi.core.DList
import pl.edu.icm.coansys.importers.models.DocumentProtos._
import pl.edu.icm.coansys.importers.models.DocumentProtosWrapper._
import com.nicta.scoobi.InputsOutputs.convertValueFromSequenceFile

/**
 * @author <NAME> (<EMAIL>)
 */
object IndexBuilder extends ScoobiApp {
  override def upload = false

  def mockReadDocs(): DList[DocumentMetadataWrapper] = DList.apply[DocumentMetadataWrapper](
    DocumentMetadata.newBuilder().setKey("1").addAuthor(Author.newBuilder().setKey("1").setName("aaa bbb")).build(),
    DocumentMetadata.newBuilder().setKey("2").addAuthor(Author.newBuilder().setKey("2").setName("bab ccc")).build(),
    DocumentMetadata.newBuilder().setKey("3").addAuthor(Author.newBuilder().setKey("3").setName("cc ddd")).build(),
    DocumentMetadata.newBuilder().setKey("4").addAuthor(Author.newBuilder().setKey("4").setName("ddd eee")).build()
  )

  def readDocsFromSeqFiles(uris: List[String]): DList[DocumentMetadataWrapper] = {
    implicit val converter = new BytesConverter[DocumentWrapper](_.toByteArray, DocumentWrapper.parseFrom(_))
    convertValueFromSequenceFile[DocumentWrapper](uris).map {
      wrapper => DocumentMetadata.parseFrom(wrapper.getMproto)
    }
  }

  def run() {
    if (args.length != 2) {
      println("Usage: IndexBuilder [-key|-author] <input_seqfile> <output_index_path>")
    } else {
      args(0) match {
        case "-key" =>
          SimpleIndex.buildKeyIndex(readDocsFromSeqFiles(List(args(0))), args(1))
        case "-author" =>
          ApproximateIndex.buildAuthorIndex(readDocsFromSeqFiles(List(args(0))), args(1))<commit_msg>small but important fixes<commit_after>package pl.edu.icm.coansys.citations

import com.nicta.scoobi.application.ScoobiApp
import com.nicta.scoobi.core.DList
import pl.edu.icm.coansys.importers.models.DocumentProtos._
import pl.edu.icm.coansys.importers.models.DocumentProtosWrapper._
import com.nicta.scoobi.InputsOutputs.convertValueFromSequenceFile

/**
 * @author <NAME> (<EMAIL>)
 */
object IndexBuilder extends ScoobiApp {
  override def upload = false

  def mockReadDocs(): DList[DocumentMetadataWrapper] = DList.apply[DocumentMetadataWrapper](
    DocumentMetadata.newBuilder().setKey("1").addAuthor(Author.newBuilder().setKey("1").setName("aaa bbb")).build(),
    DocumentMetadata.newBuilder().setKey("2").addAuthor(Author.newBuilder().setKey("2").setName("bab ccc")).build(),
    DocumentMetadata.newBuilder().setKey("3").addAuthor(Author.newBuilder().setKey("3").setName("cc ddd")).build(),
    DocumentMetadata.newBuilder().setKey("4").addAuthor(Author.newBuilder().setKey("4").setName("ddd eee")).build()
  )

  def readDocsFromSeqFiles(uris: List[String]): DList[DocumentMetadataWrapper] = {
    implicit val converter = new BytesConverter[DocumentWrapper](_.toByteArray, DocumentWrapper.parseFrom(_))
    convertValueFromSequenceFile[DocumentWrapper](uris).map {
      wrapper => DocumentMetadata.parseFrom(wrapper.getMproto)
    }
  }

  def run() {
    if (args.length != 3) {
      println("Usage: IndexBuilder [-key|-author] <input_seqfile> <output_index_path>")
    } else {
      args(0) match {
        case "-key" =>
          SimpleIndex.buildKeyIndex(readDocsFromSeqFiles(List(args(1))), args(2))
        case "-author" =>
          ApproximateIndex.buildAuthorIndex(readDocsFromSeqFiles(List(args(1))), args(2))
{'fraction_non_alphanumeric': 0.1535796766743649, 'fraction_numerical': 0.008083140877598153, 'mean_word_length': 5.322992700729927, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5453307', 'n_tokens_mistral': 1121, 'n_tokens_neox': 1095, 'n_words': 140}