<commit_before>package org.wow.learning.vectorizers.planet

import org.wow.learning.vectorizers.Vectorizer
import org.apache.mahout.math.Vector
import com.epam.starwors.galaxy.Planet
import org.wow.evaluation.transition.PlayerGameTurn
import org.wow.logger.PlayerMove
import org.apache.mahout.vectorizer.encoders.FeatureVectorEncoder
import org.apache.mahout.math.RandomAccessSparseVector

public data class PlanetTransition(val from: Planet,
                                   val to: Planet,
                                   val moves: List<PlayerMove>)

data class FeatureExtractor(val encoder: FeatureVectorEncoder,val eval: (Planet) -> Double, val weight: Double)

fun planetMoves(transition: PlayerGameTurn): List<PlanetTransition> {
    val planetsThatMoves = transition.from.moves.fold(listOf<String>(), { (acc, item) ->
        acc.plus(item.from).plus(item.to)}).toSet()
    return transition.from.planets
            .filter { it.getOwner() == transition.playerName  }
            .filter { planetsThatMoves.contains(it.getId()) }
            .map { sourcePlanet -> PlanetTransition(sourcePlanet,
                    transition.to.planets.first { it.getId() == sourcePlanet.getId() },
                    transition.from.moves) }
}


public class PlanetVectorizer(private val featuresExtractors: List<FeatureExtractor>) : Vectorizer<Planet,
        Vector> {

    override fun vectorize(input: Planet): Vector {
        return featuresExtractors.withIndices()
                .fold<Pair<Int, FeatureExtractor>, Vector>(RandomAccessSparseVector(featuresExtractors.size),
                        { (acc, extractor) ->
//                            acc.set(extractor.first, extractor.second.eval(input) * extractor.second.weight); acc })
                            extractor.second.encoder.addToVector(null: ByteArray?,
                                    extractor.second.eval(input) * extractor.second.weight,
                                    acc); acc })
    }
}<commit_msg>Training bots algo contains lot of symmetric moves. Filtered<commit_after>package org.wow.learning.vectorizers.planet

import com.epam.starwors.galaxy.Planet
import org.wow.evaluation.transition.PlayerGameTurn
import org.wow.logger.PlayerMove
import org.apache.mahout.vectorizer.encoders.FeatureVectorEncoder
import org.wow.learning.vectorizers.Vectorizer
import org.apache.mahout.math.Vector
import org.apache.mahout.math.RandomAccessSparseVector

public data class PlanetTransition(val from: Planet,
                                   val to: Planet,
                                   val moves: List<PlayerMove>)

data class FeatureExtractor(val encoder: FeatureVectorEncoder,val eval: (Planet) -> Double, val weight: Double)

fun planetMoves(transition: PlayerGameTurn): List<PlanetTransition> {
    val nonSymmetricMoves = transition.from.moves.fold(listOf<PlayerMove>(), {(acc, move) ->
        if(transition.from.moves.any { it.from == move.to &&
                it.to == move.from && it.unitCount > move.unitCount })
            acc else acc.plus(move)
    })
    val planetsThatMoves = nonSymmetricMoves.fold(listOf<String>(), { (acc, item) ->
        acc.plus(item.from).plus(item.to)}).toSet()
    return transition.from.planets
            .filter { it.getOwner() == transition.playerName  }
            .filter { planetsThatMoves.contains(it.getId()) }
            .map { sourcePlanet -> PlanetTransition(sourcePlanet,
                    transition.to.planets.first { it.getId() == sourcePlanet.getId() },
                    transition.from.moves) }
}


public class PlanetVectorizer(private val featuresExtractors: List<FeatureExtractor>) : Vectorizer<Planet,
        Vector> {

    override fun vectorize(input: Planet): Vector {
        return featuresExtractors.withIndices()
                .fold<Pair<Int, FeatureExtractor>, Vector>(RandomAccessSparseVector(featuresExtractors.size),
                        { (acc, extractor) ->
                            acc.set(extractor.first, extractor.second.eval(input) * extractor.second.weight); acc })
    }
}
{'fraction_non_alphanumeric': 0.10843373493975904, 'fraction_numerical': 0.0, 'mean_word_length': 2.7184643510054847, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5937700', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1119, 'n_words': 203}