<commit_before>// Copyright 2015 <NAME>. All rights reserved.
// Proprietary license.

// Package translator
package translator
import (
	"net/http"
	"time"
	"github.com/facebookgo/httpcontrol"
	"sync"
	"log"
//	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/translation_middleware"
	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/particular"
	"regexp"
)

// Translator interface
type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	AddParticular(p particular.IParticularBackend)
}

var (
	client *http.Client
	once = &sync.Once{}
)

func initClient() (*http.Client) {
	once.Do(func() {
		transport := &httpcontrol.Transport{
			RequestTimeout: time.Minute,
			MaxTries: 3,
			MaxIdleConnsPerHost: 10,
		}

		client = &http.Client{
			Transport: transport,
		}
	})

	return client
}

type IBackendResponse interface {
	GetText() string
	GetSource() string
	GetLang() string
}

type ITranslateBackend interface {
	TranslateOne(text string, language string) (data IBackendResponse)
	GetName() string
}

type TranslateAdapter struct {
	client *http.Client
	translateBackend []ITranslateBackend
	translateParticular []particular.IParticularBackend
//	middleware []translation_middleware.ITranslationMiddleware
	landChan chan string

}

func NewTranslateAdapter(back []ITranslateBackend) *TranslateAdapter {
	return &TranslateAdapter{client:initClient(), translateBackend:back, landChan:make(chan string, 1)}
}
func (t *TranslateAdapter) AddParticular(p particular.IParticularBackend){
	p.SetClient(t.client)
	t.translateParticular = append(t.translateParticular, p)
}
func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	container := t.newContainer()
	container.Original = text

	langs = array_uniq(langs)
	processor := NewEmojiProcessor()
	text = processor.Process(text)

	responseChan := make(chan *RawTransData, len(langs))
	responseChanParticular := make(chan *RawParticularData, len(langs))

	go t.doRequestsTranslators(text, langs, responseChan)
	go func(){
		lang := <-t.landChan
		t.doRequestsParticular(text, lang, langs, responseChanParticular)
	}()
	for resp := range responseChan {
		log.Println(resp)
		container.RawTranslations[resp.Name][resp.Lang] = resp.Translation
		if "google" == resp.Name {
			container.Source = resp.Source
			t.landChan <- resp.Source
		}
		resp.Translation = processor.Restore(resp.Translation)
		container.RawTransData = append(container.RawTransData, resp)
	}
	container.Translations = container.RawTranslations["google"]

	for resp := range responseChanParticular {
		container.RawParticularData = append(container.RawParticularData, resp)
	}
	return container
}

func (t *TranslateAdapter) doRequestsTranslators(text string, languages []string, c chan<- *RawTransData){

	wg := &sync.WaitGroup{}
	for _, v := range languages {
		for _, back := range t.translateBackend{
			wg.Add(1)
			go func(text, lang string, backend ITranslateBackend){
				defer wg.Done()
				t := time.Now()
				resp := backend.TranslateOne(text, lang)

				raw := &RawTransData{
					Source:resp.GetSource(),
					Lang:resp.GetLang(),
					Name:backend.GetName(),
					Translation:resp.GetText(),
					Time: time.Since(t) / time.Millisecond,
				}
				c <- raw
			}(text, v, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) doRequestsParticular(text, source string, languages []string, c chan<- *RawParticularData){

	bag := regexp.MustCompile(`([\w\x7f-\xff])+`).FindAllString(text, -1)

	wg := &sync.WaitGroup{}
	for _, lang := range languages {
		for _, back := range t.translateParticular{
			wg.Add(1)
			raw := &RawParticularData{
				Original: text,<commit_msg>em fix<commit_after>// Copyright 2015 <NAME>. All rights reserved.
// Proprietary license.

// Package translator
package translator
import (
	"net/http"
	"time"
	"github.com/facebookgo/httpcontrol"
	"sync"
	"log"
//	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/translation_middleware"
	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/particular"
	"regexp"
)

// Translator interface
type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	AddParticular(p particular.IParticularBackend)
}

var (
	client *http.Client
	once = &sync.Once{}
)

func initClient() (*http.Client) {
	once.Do(func() {
		transport := &httpcontrol.Transport{
			RequestTimeout: time.Minute,
			MaxTries: 3,
			MaxIdleConnsPerHost: 10,
		}

		client = &http.Client{
			Transport: transport,
		}
	})

	return client
}

type IBackendResponse interface {
	GetText() string
	GetSource() string
	GetLang() string
}

type ITranslateBackend interface {
	TranslateOne(text string, language string) (data IBackendResponse)
	GetName() string
}

type TranslateAdapter struct {
	client *http.Client
	translateBackend []ITranslateBackend
	translateParticular []particular.IParticularBackend
//	middleware []translation_middleware.ITranslationMiddleware
	landChan chan string

}

func NewTranslateAdapter(back []ITranslateBackend) *TranslateAdapter {
	return &TranslateAdapter{client:initClient(), translateBackend:back, landChan:make(chan string, 1)}
}
func (t *TranslateAdapter) AddParticular(p particular.IParticularBackend){
	p.SetClient(t.client)
	t.translateParticular = append(t.translateParticular, p)
}
func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	container := t.newContainer()
	container.Original = text

	langs = array_uniq(langs)
	processor := NewEmojiProcessor()
	text = processor.Process(text)

	responseChan := make(chan *RawTransData, len(langs))
	responseChanParticular := make(chan *RawParticularData, len(langs))

	go t.doRequestsTranslators(text, langs, responseChan)
	go func(){
		lang := <-t.landChan
		t.doRequestsParticular(text, lang, langs, responseChanParticular)
	}()
	for resp := range responseChan {
		log.Println(resp)
		container.RawTranslations[resp.Name][resp.Lang] = resp.Translation
		if "google" == resp.Name {
			container.Source = resp.Source
			t.landChan <- resp.Source
		}
		resp.Translation = processor.Restore(resp.Translation)
		container.RawTransData = append(container.RawTransData, resp)
	}
	container.Translations = container.RawTranslations["google"]

	for resp := range responseChanParticular {
		container.RawParticularData = append(container.RawParticularData, resp)
	}
	return container
}

func (t *TranslateAdapter) doRequestsTranslators(text string, languages []string, c chan<- *RawTransData){
	emFix := regexp.MustCompile(`\_\s?\$\s?(\d+)\_+`)
	wg := &sync.WaitGroup{}
	for _, v := range languages {
		for _, back := range t.translateBackend{
			wg.Add(1)
			go func(text, lang string, backend ITranslateBackend){
				defer wg.Done()
				t := time.Now()
				resp := backend.TranslateOne(text, lang)

				raw := &RawTransData{
					Source:resp.GetSource(),
					Lang:resp.GetLang(),
					Name:backend.GetName(),
					Translation:emFix.ReplaceAllString(resp.GetText(), `_\$$1_`),
					Time: time.Since(t) / time.Millisecond,
				}
				c <- raw
			}(text, v, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) doRequestsParticular(text, source string, languages []string, c chan<- *RawParticularData){

	bag := regexp.MustCompile(`([\w\x7f-\xff])+`).FindAllString(text, -1)

	wg := &sync.WaitGroup{}
	for _, lang := range languages {
		for _, back := range t.translateParticular{
			wg.Add(1)
			raw := &RawParticularData{
				Original: text,
{'fraction_non_alphanumeric': 0.12701586032253764, 'fraction_numerical': 0.0033320005331200853, 'mean_word_length': 4.876272513703994, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5281172', 'n_tokens_mistral': 2689, 'n_tokens_neox': 2458, 'n_words': 517}