<commit_before>package gogadgets

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
	"time"

	"github.com/cswank/rex"
)

type Server struct {
	host        string
	master      string
	isMaster    bool
	port        int
	prefix      string
	lg          Logger
	external    chan Message
	internal    chan Message
	id          string
	updates     map[string]Message
	seen        map[string]time.Time
	statusLock  sync.Mutex
	seenLock    sync.Mutex
	clientsLock sync.Mutex
	clients     map[string]string
}

func NewServer(host, master string, port int, lg Logger) *Server {
	var isMaster bool
	clients := map[string]string{}
	if master == "" {
		isMaster = true
	} else {
		clients = map[string]string{
			master: "",
		}
	}
	return &Server{
		master:   master,
		host:     host,
		isMaster: isMaster,
		port:     port,
		lg:       lg,
		updates:  map[string]Message{},
		id:       "server",
		external: make(chan Message),
		seen:     map[string]time.Time{},
		clients:  clients,
	}
}

func (s *Server) Start(i <-chan Message, o chan<- Message) {
	if !s.isMaster {
		go s.register()
	}
	go s.startServer()
	go s.cleanup()

	for {
		select {
		case msg := <-i:
			if (msg.Type == UPDATE || msg.Type == METHODUPDATE) && s.isMaster {
				s.statusLock.Lock()
				s.updates[msg.Sender] = msg
				s.statusLock.Unlock()
			}
			if !s.isSeen(msg) {
				s.send(msg)
			}
		case msg := <-s.external:
			s.setSeen(msg)
			o <- msg
			if s.isMaster && msg.Sender == "client" {
				s.send(msg)
			}
		}
	}
}

func (s *Server) send(msg Message) {
	s.clientsLock.Lock()
	for host, token := range s.clients {
		go s.doSend(host, msg, token)
	}
	s.clientsLock.Unlock()
}

func (s *Server) doSend(host string, msg Message, token string) {
	msg.Host = s.host
	var buf bytes.Buffer
	enc := json.NewEncoder(&buf)
	enc.Encode(msg)
	req, _ := http.NewRequest("POST", host, &buf)
	if len(token) > 0 {
		req.Header.Add("Authorization", "Bearer "+token)
	}
	r, err := http.DefaultClient.Do(req)
	if r != nil {
		r.Body.Close()
	}

	if err != nil || r.StatusCode != http.StatusOK {
		s.clientsLock.Lock()
		delete(s.clients, host)
		s.clientsLock.Unlock()
	}
}

func (s *Server) setSeen(msg Message) {
	s.seenLock.Lock()
	s.seen[msg.UUID] = time.Now()
	s.seenLock.Unlock()
}
<commit_msg>handle invalid host when posting to quimby<commit_after>package gogadgets

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/cswank/rex"
)

type Server struct {
	host        string
	master      string
	isMaster    bool
	port        int
	prefix      string
	lg          Logger
	external    chan Message
	internal    chan Message
	id          string
	updates     map[string]Message
	seen        map[string]time.Time
	statusLock  sync.Mutex
	seenLock    sync.Mutex
	clientsLock sync.Mutex
	clients     map[string]string
}

func NewServer(host, master string, port int, lg Logger) *Server {
	var isMaster bool
	clients := map[string]string{}
	if master == "" {
		isMaster = true
	} else {
		clients = map[string]string{
			master: "",
		}
	}
	return &Server{
		master:   master,
		host:     host,
		isMaster: isMaster,
		port:     port,
		lg:       lg,
		updates:  map[string]Message{},
		id:       "server",
		external: make(chan Message),
		seen:     map[string]time.Time{},
		clients:  clients,
	}
}

func (s *Server) Start(i <-chan Message, o chan<- Message) {
	if !s.isMaster {
		go s.register()
	}
	go s.startServer()
	go s.cleanup()

	for {
		select {
		case msg := <-i:
			if (msg.Type == UPDATE || msg.Type == METHODUPDATE) && s.isMaster {
				s.statusLock.Lock()
				s.updates[msg.Sender] = msg
				s.statusLock.Unlock()
			}
			if !s.isSeen(msg) {
				s.send(msg)
			}
		case msg := <-s.external:
			s.setSeen(msg)
			o <- msg
			if s.isMaster && msg.Sender == "client" {
				s.send(msg)
			}
		}
	}
}

func (s *Server) send(msg Message) {
	s.clientsLock.Lock()
	for host, token := range s.clients {
		go s.doSend(host, msg, token)
	}
	s.clientsLock.Unlock()
}

func (s *Server) doSend(host string, msg Message, token string) {
	msg.Host = s.host
	var buf bytes.Buffer
	enc := json.NewEncoder(&buf)
	enc.Encode(msg)
	req, err := http.NewRequest("POST", host, &buf)
	if err != nil {
		log.Printf("error posting to quimby host: %s, err: %v\n", host, err)
		s.clientsLock.Lock()
		delete(s.clients, host)
		s.clientsLock.Unlock()
		return
	}
	if len(token) > 0 {
		req.Header.Add("Authorization", "Bearer "+token)
	}
	r, err := http.DefaultClient.Do(req)
	if r != nil {
		r.Body.Close()
	}

	if err != nil || r.StatusCode != http.StatusOK {
		s.clientsLock.Lock()
		delete(s.clients, host)
		s.clientsLock.Unlock()
	}
}

func (s *Server) setSeen(msg Message) {
	s.seenLock.Lock()
	s.seen[msg.UUID] = time.Now()
	s.seenLock.Unlock()
}

{'fraction_non_alphanumeric': 0.1468605141171513, 'fraction_numerical': 0.00042140750105351877, 'mean_word_length': 2.825141015310234, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1800674', 'n_tokens_mistral': 1983, 'n_tokens_neox': 1844, 'n_words': 434}