<commit_before>package ch.uzh.ifi.pdeboer.pplib.examples.textshortening

import ch.uzh.ifi.pdeboer.pplib.process.entities._
import ch.uzh.ifi.pdeboer.pplib.process.recombination.{RecombinationHints, RecombinationVariantGenerator, Recombinator}

/**
 * Created by pdeboer on 12/05/15.
 */
object ShortNText extends App {
	val testData = new ShortNTestDataInitializer()
	testData.initializePortal()

	val recombinator = new Recombinator(RecombinationHints.create(Map(
		RecombinationHints.DEFAULT_HINTS -> {
			RecombinationHints.hcompPlatform(List(testData.hcompPortalWithMockAnswers)) :::
				RecombinationHints.instructions(List(
					new InstructionData(actionName = "shorten the following paragraph", detailedDescription = "grammar (e.g. tenses), text-length")))
		})
	))

	val processes = Map("shortener" -> recombinator.materialize[CreateProcess[_ <: List[Patch], _ <: List[Patch]]])

	val surfaceStructure = new ShortNSurfaceStructure(testData.text)
	val results = new RecombinationVariantGenerator(processes).variants.map(variant => {
		(variant, surfaceStructure.runRecombinedVariant(variant))
	})

	println(results)
}<commit_msg>print shortest results<commit_after>package ch.uzh.ifi.pdeboer.pplib.examples.textshortening

import ch.uzh.ifi.pdeboer.pplib.process.entities._
import ch.uzh.ifi.pdeboer.pplib.process.recombination.{RecombinationHints, RecombinationVariantGenerator, Recombinator}

/**
 * Created by pdeboer on 12/05/15.
 */
object ShortNText extends App {
	val testData = new ShortNTestDataInitializer()
	testData.initializePortal()

	val recombinator = new Recombinator(RecombinationHints.create(Map(
		RecombinationHints.DEFAULT_HINTS -> {
			RecombinationHints.hcompPlatform(List(testData.hcompPortalWithMockAnswers)) :::
				RecombinationHints.instructions(List(
					new InstructionData(actionName = "shorten the following paragraph", detailedDescription = "grammar (e.g. tenses), text-length")))
		})
	))

	val processes = Map("shortener" -> recombinator.materialize[CreateProcess[_ <: List[Patch], _ <: List[Patch]]])

	val surfaceStructure = new ShortNSurfaceStructure(testData.text)
	val results = new RecombinationVariantGenerator(processes).variants.map(variant => {
		(variant, surfaceStructure.runRecombinedVariant(variant))
	})

	println(s"shortest result: ${results.minBy(_._2.length)}")

	println("all results")
	println(results)
}
{'fraction_non_alphanumeric': 0.11687763713080168, 'fraction_numerical': 0.005485232067510549, 'mean_word_length': 6.273006134969325, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2408195', 'n_tokens_mistral': 822, 'n_tokens_neox': 758, 'n_words': 121}