<commit_before>package es.upm.oeg.epnoi.matching.metrics.corpus

import es.upm.oeg.epnoi.matching.metrics.feature.LuceneTokenizer
import es.upm.oeg.epnoi.matching.metrics.model._
import es.upm.oeg.epnoi.matching.metrics.space.{Metadata, Resource}
import es.upm.oeg.epnoi.matching.metrics.utils.SparkWrapper

/**
 * Corpus defined by 4 topics: baseball, motor, religion and space
 */
case object Articles {

  val corpus = SparkWrapper.readCorpus("src/test/corpus/articles/*").map{x=>
    val name = x._1.substring(x._1.lastIndexOf("/")+1)
    Resource(
      uri         = s"ro.oeg.es/resource/$name",
      url         = x._1,
      metadata    = Metadata(s"title-$name","2011",Some(List(Authors.a1,Authors.a2,Authors.a3))),
      words       = Some(LuceneTokenizer(x._2)),
      resources   = None)
  }

 }
<commit_msg>groupBy error using Array fixed for TopicModels<commit_after>package es.upm.oeg.epnoi.matching.metrics.corpus

import es.upm.oeg.epnoi.matching.metrics.feature.LuceneTokenizer
import es.upm.oeg.epnoi.matching.metrics.model._
import es.upm.oeg.epnoi.matching.metrics.space.{Metadata, Resource}
import es.upm.oeg.epnoi.matching.metrics.utils.SparkWrapper

/**
 * Corpus defined by 4 topics: baseball, motor, religion and space
 */
case object Articles {

  val corpus = SparkWrapper.readCorpus("src/test/corpus/articles/*.txt").map{x=>
    val name = x._1.substring(x._1.lastIndexOf("/")+1)
    Resource(
      uri         = s"ro.oeg.es/resource/$name",
      url         = x._1,
      metadata    = Metadata(s"title-$name","2011",Some(List(Authors.a1,Authors.a2,Authors.a3))),
      words       = Some(LuceneTokenizer(x._2)),
      resources   = None)
  }

 }

{'fraction_non_alphanumeric': 0.14702380952380953, 'fraction_numerical': 0.015476190476190477, 'mean_word_length': 3.400523560209424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1566528', 'n_tokens_mistral': 630, 'n_tokens_neox': 593, 'n_words': 88}