<commit_before>package main

import (
	// stdlib packages

	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"strconv"
	"sync"

	// custom packages
	"config"
	"modules/amqpmodule"
	es "modules/elasticsearchmodule"
)

const rxQueue = "conn_scan_results_queue"
const rxRoutKey = "conn_scan_results"
const esIndex = "observer"
const esType = "connection"

var broker *amqpmodule.Broker

//the 2 following structs represent the cipherscan output.

type ScanInfo struct {
	Target         string        `json:"target"`
	Timestamp      string        `json:"utctimestamp"`
	ServerSide     string        `json:"serverside"`
	CurvesFallback string        `json:"curves_fallback"`
	CipherSuites   []Ciphersuite `json:"ciphersuite"`
}

type Ciphersuite struct {
	Cipher       string   `json:"cipher"`
	Protocols    []string `json:"protocols"`
	PubKey       []string `json:"pubkey"`
	SigAlg       []string `json:"sigalg"`
	Trusted      string   `json:"trusted"`
	TicketHint   string   `json:"ticket_hint"`
	OCSPStapling string   `json:"ocsp_stapling"`
	PFS          string   `json:"pfs"`
	Curves       []string `json:"curves,omitempty"`
}

//the following structs represent the output we want to provide to DB.

type ConnectionInfo struct {
	ConnectionTimestamp string                           `json:"connectionTimestamp"`
	ServerSide          bool                             `json:"serverside"`
	CipherSuites        map[string]ConnectionCiphersuite `json:"ciphersuite"`
	CurvesFallback      bool                             `json:"curvesFallback"`
}

type ConnectionCiphersuite struct {
	Cipher       string   `json:"cipher"`
	Protocols    []string `json:"protocols"`
	PubKey       float64  `json:"pubkey"`
	SigAlg       string   `json:"sigalg"`
	TicketHint   string   `json:"ticket_hint"`
	OCSPStapling bool     `json:"ocsp_stapling"`
	PFS          string   `json:"pfs"`
	Curves       []string `json:"curves"`
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func panicIf(err error) bool {
	if err != nil {
		log.Println(fmt.Sprintf("%s", err))
		return true
	}

	return false
}

func stringtoBool(s string) bool {
	if s == "True" {
		return true
	} else {
		return false
	}

}

func (s ScanInfo) toConnInfo() (ConnectionInfo, error) {

	c := ConnectionInfo{}

	var err error

	c.ConnectionTimestamp = s.Timestamp
	c.ServerSide = stringtoBool(s.ServerSide)
	c.CurvesFallback = stringtoBool(s.CurvesFallback)

	c.CipherSuites = make(map[string]ConnectionCiphersuite)

	pos := 1

	for _, cipher := range s.CipherSuites {

		newcipher := ConnectionCiphersuite{}

		newcipher.Cipher = cipher.Cipher
		newcipher.OCSPStapling = stringtoBool(cipher.OCSPStapling)
		newcipher.PFS = cipher.PFS

		newcipher.Protocols = cipher.Protocols

		if len(cipher.PubKey) > 1 {
			log.Println("Multiple PubKeys for ", s.Target, " at cipher :", cipher.Cipher)
		}

		if len(cipher.PubKey) > 0 {
			newcipher.PubKey, err = strconv.ParseFloat(cipher.PubKey[0], 64)
		} else {
			return c, fmt.Errorf("No Public Keys found")
		}

		if len(cipher.SigAlg) > 1 {
			log.Println("Multiple SigAlgs for ", s.Target, " at cipher :", cipher.Cipher)
		}

		if len(cipher.SigAlg) > 0 {
			newcipher.SigAlg = cipher.SigAlg[0]
		} else {
			return c, fmt.Errorf("No Signature Algorithms found")
		}

		newcipher.TicketHint = cipher.TicketHint

		if err != nil {
			return c, err
		}

		newcipher.Curves = append(newcipher.Curves, cipher.Curves...)

		c.CipherSuites[strconv.Itoa(pos)] = newcipher
		pos++
	}

	return c, nil

}

//worker is the main body of the goroutine that handles each received message.
func worker(msgs <-chan []byte) {

	forever := make(chan bool)
	defer wg.Done()

	for d := range msgs {

		info := ScanInfo{}

		err := json.Unmarshal(d, &info)

		panicIf(err)

		if err != nil {
			continue
		}

		c, err := info.toConnInfo()

		panicIf(err)

		if err != nil {
			continue
		}

		id := info.Target

		jsonConn, err := json.Marshal(c)

		panicIf(err)

		if err != nil {
			continue
		}

		err = es.Push(esIndex, esType, id, jsonConn)
		panicIf(err)
	}

	<-forever
}

func printIntro() {
	fmt.Println(`
	##################################
	#         TLSAnalyzer            #
	##################################
	`)
}

var wg sync.WaitGroup

func main() {
	var (
		err error
	)

	printIntro()

	conf := config.AnalyzerConfig{}

	var cfgFile string
	flag.StringVar(&cfgFile, "c", "/etc/observer/analyzer.cfg", "Input file csv format")
	flag.Parse()

	_, err = os.Stat(cfgFile)
	failOnError(err, "Missing configuration file from '-c' or /etc/observer/retriever.cfg")<commit_msg>moved connection structs to separate package, implemented target connection versioning...<commit_after>package main

import (
	// stdlib packages

	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"sync"

	// custom packages
	"config"
	"connection"
	"modules/amqpmodule"

	es "modules/elasticsearchmodule"
)

const rxQueue = "conn_scan_results_queue"
const rxRoutKey = "conn_scan_results"
const esIndex = "observer"
const esType = "connection"

var broker *amqpmodule.Broker

//the 2 following structs represent the cipherscan output.

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func panicIf(err error) bool {
	if err != nil {
		log.Println(fmt.Sprintf("%s", err))
		return true
	}

	return false
}

// retrieves stored connections ( if any ) for the given scan target
func getConnsforTarget(t string) (map[string]connection.Stored, error) {

	res, err := es.SearchbyTerm(esIndex, esType, "scanTarget.raw", t)

	if err != nil {
		return nil, err
	}

	storedConns := make(map[string]connection.Stored)

	if res.Total > 0 && err != nil {

		for i := 0; i < res.Total; i++ {

			s := connection.Stored{}
			err = json.Unmarshal(*res.Hits[i].Source, &s)

			if err != nil {
				panicIf(err)
				continue
			}

			storedConns[res.Hits[i].Id] = s
		}

		if len(storedConns) > 0 {
			return storedConns, nil
		}
	}

	return storedConns, nil
}

//worker is the main body of the goroutine that handles each received message.
func worker(msgs <-chan []byte) {

	forever := make(chan bool)
	defer wg.Done()

	for d := range msgs {

		info := connection.CipherscanOutput{}

		err := json.Unmarshal(d, &info)

		panicIf(err)

		if err != nil {
			continue
		}

		c, err := info.Stored()

		panicIf(err)
		if err != nil {
			continue
		}

		stored, err := getConnsforTarget(c.ScanTarget)

		if err != nil {
			panicIf(err)
			continue
		}

		err = updateAndPushConnections(c, stored)

		panicIf(err) //Should we requeue the connection in case of error?
	}

	<-forever
}

func updateAndPushConnections(newconn connection.Stored, conns map[string]connection.Stored) error {

	err := error(nil)

	if len(conns) > 0 {
		for id, conn := range conns {
			if conn.ObsoletedBy == "" {
				if newconn.Equal(conn) {

					log.Println("Updating doc for ", conn.ScanTarget)
					conn.LastSeenTimestamp = newconn.LastSeenTimestamp

					jsonConn, err := json.Marshal(conn)

					if err == nil {
						_, err = es.Push(esIndex, esType, "", jsonConn)
					}

					break

				} else {

					log.Println("Pushing new doc for ", conn.ScanTarget)

					jsonConn, err := json.Marshal(newconn)

					obsID := ""

					if err != nil {
						break
					}

					obsID, err = es.Push(esIndex, esType, "", jsonConn)

					if err != nil {
						break
					}

					conn.ObsoletedBy = obsID

					jsonConn, err = json.Marshal(conn)

					obsID, err = es.Push(esIndex, esType, id, jsonConn)
				}
			}
		}
	} else {

		log.Println("No older doc found for ", newconn.ScanTarget)

		jsonConn, err := json.Marshal(newconn)

		if err == nil {
			_, err = es.Push(esIndex, esType, "", jsonConn)
		}

	}

	return err
}

func printIntro() {
	fmt.Println(`
	##################################
	#         TLSAnalyzer            #
	##################################
	`)
}

var wg sync.WaitGroup

func main() {
	var (
		err error
	)

	printIntro()

	conf := config.AnalyzerConfig{}

	var cfgFile string
	flag.StringVar(&cfgFile, "c", "/etc/observer/analyzer.cfg", "Input file csv format")
	flag.Parse()

	_, err = os.Stat(cfgFile)
	failOnError(err, "Missing configuration file from '-c' or /etc/observer/retriever.cfg")
{'fraction_non_alphanumeric': 0.14290865384615384, 'fraction_numerical': 0.002043269230769231, 'mean_word_length': 3.1542685971043434, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3373120', 'n_tokens_mistral': 3259, 'n_tokens_neox': 2883, 'n_words': 712}