<commit_before>// Copyright 2016 Home24 AG. All rights reserved.
// Proprietary license.
package translator

import (
	"github.com/urakozz/transpoint.io/src/components"
	"github.com/urakozz/transpoint.io/src/translator/backend_full"
	"github.com/urakozz/transpoint.io/src/translator/backend_particular"
	"github.com/urakozz/transpoint.io/src/translator/processing"
	"math"
	"strings"
	"sync"
	"time"
)

type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	//AddParticular(p particular.IParticularBackend)
}

type TranslateAdapter struct {
	translateBackend    []backend_full.IBackendFull
	translateParticular []backend_particular.IBackendParticular
	markov              components.IChain
	landChan            chan string
}

func NewTranslateAdapter(back []backend_full.IBackendFull, markov components.IChain) *TranslateAdapter {
	return &TranslateAdapter{translateBackend: back, landChan: make(chan string, 1), markov: markov}
}

func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	langs = array_uniq(langs)

	container := t.createContainer(text, langs)

	//fill markov model
	if len(text) > 100 {
		t.markov.Add(text)
	}

	segments := processing.Segments.Split(text)
	container.RawSegmentsData = t.GetSegmentTranslations(segments, langs)
	t.CalculateScores(container, segments)
	t.AppendUniEngine(container, segments)
	t.CalculateRawTransData(container)

	for lang, details := range container.RawTransData {
		container.Translations[lang] = details["uni"].Translation
	}

	return container
}

func (t *TranslateAdapter) CalculateScores(container *TranslationContainer, segments []*processing.Segment) {
	for i, seg := range segments {
		if seg.Type > processing.SegmentText {
			continue
		}
		for _, lang := range container.Langs {
			for _, back := range t.translateBackend {
				rawSegData := container.RawSegmentsData[i][lang][back.GetName()]
				rawSegData.Score = t.markov.Occurrences(rawSegData.Translation)
			}
		}
	}
}

func (t *TranslateAdapter) AppendUniEngine(container *TranslationContainer, segments []*processing.Segment) {

	for i, seg := range segments {
		for _, lang := range container.Langs {<commit_msg>lang container<commit_after>// Copyright 2016 Home24 AG. All rights reserved.
// Proprietary license.
package translator

import (
	"github.com/urakozz/transpoint.io/src/components"
	"github.com/urakozz/transpoint.io/src/translator/backend_full"
	"github.com/urakozz/transpoint.io/src/translator/backend_particular"
	"github.com/urakozz/transpoint.io/src/translator/processing"
	"math"
	"strings"
	"sync"
	"time"
)

type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	//AddParticular(p particular.IParticularBackend)
}

type TranslateAdapter struct {
	translateBackend    []backend_full.IBackendFull
	translateParticular []backend_particular.IBackendParticular
	markov              components.IChain
	landChan            chan string
}

func NewTranslateAdapter(back []backend_full.IBackendFull, markov components.IChain) *TranslateAdapter {
	return &TranslateAdapter{translateBackend: back, landChan: make(chan string, 1), markov: markov}
}

func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	langs = array_uniq(langs)

	container := t.createContainer(text, langs)

	//fill markov model
	if len(text) > 100 {
		t.markov.Add(text)
	}

	segments := processing.Segments.Split(text)
	container.RawSegmentsData = t.GetSegmentTranslations(segments, langs)
	t.CalculateScores(container, segments)
	t.AppendUniEngine(container, segments)
	t.CalculateRawTransData(container)

	for lang, details := range container.RawTransData {
		container.Translations[lang] = details["uni"].Translation
		container.Source = details["uni"].Source
	}

	return container
}

func (t *TranslateAdapter) CalculateScores(container *TranslationContainer, segments []*processing.Segment) {
	for i, seg := range segments {
		if seg.Type > processing.SegmentText {
			continue
		}
		for _, lang := range container.Langs {
			for _, back := range t.translateBackend {
				rawSegData := container.RawSegmentsData[i][lang][back.GetName()]
				rawSegData.Score = t.markov.Occurrences(rawSegData.Translation)
			}
		}
	}
}

func (t *TranslateAdapter) AppendUniEngine(container *TranslationContainer, segments []*processing.Segment) {

	for i, seg := range segments {
		for _, lang := range container.Langs {
{'fraction_non_alphanumeric': 0.1101390758187528, 'fraction_numerical': 0.004486316733961417, 'mean_word_length': 5.150344827586207, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5498194', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1392, 'n_words': 285}