<commit_before>//certAnalyser provides a client that receives certificates from a queue, processes them and
//indexes them in an ElasticSearch database. The Certificate struct is used as the storage document template
package certificate

import (
	// stdlib packages
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"io/ioutil"

	"github.com/Sirupsen/logrus"

	"github.com/mozilla/tls-observatory/config"
	pg "github.com/mozilla/tls-observatory/database"
	"github.com/mozilla/tls-observatory/logger"
)

var trustStores []TrustStore
var log = logger.GetLogger()

var TS = []string{ubuntu_TS_name, mozilla_TS_name, microsoft_TS_name, apple_TS_name, android_TS_name}

func Setup(c config.Config, DB *pg.DB) {
	ts := c.TrustStores

	db = DB

	for _, name := range TS {

		path := ""

		switch name {
		case ubuntu_TS_name:

			path = ts.Ubuntu_TS

		case mozilla_TS_name:

			path = ts.Mozilla_TS

		case microsoft_TS_name:

			path = ts.Microsoft_TS

		case apple_TS_name:
			path = ts.Apple_TS

		case android_TS_name:
			path = ts.Android_TS

		default:

			log.WithFields(logrus.Fields{
				"tsname": name,
			}).Warning("Wrong slice name entry. This should not be there")
		}

		// load the entire trustore into pooldata, then iterate over each PEM block
		// until all of pooldata is read
		poolData, err := ioutil.ReadFile(path)
		if err != nil {
			log.WithFields(logrus.Fields{
				"tsname": name,
				"error":  err.Error(),
			}).Warning("Failed to load truststore")
		}
		certPool := x509.NewCertPool()
		poollen := 0
		for len(poolData) > 0 {
			// read the next PEM block, ignore non CERTIFICATE entires
			var block *pem.Block
			block, poolData = pem.Decode(poolData)
			if block == nil {
				break
			}
			if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
				continue
			}
			// parse the current PEM block into a certificate, ignore failures
			cert, err := x509.ParseCertificate(block.Bytes)
			if err != nil {
				log.WithFields(logrus.Fields{<commit_msg> added debug logging of parsed truststore and its path.<commit_after>//certAnalyser provides a client that receives certificates from a queue, processes them and
//indexes them in an ElasticSearch database. The Certificate struct is used as the storage document template
package certificate

import (
	// stdlib packages
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"io/ioutil"

	"github.com/Sirupsen/logrus"

	"github.com/mozilla/tls-observatory/config"
	pg "github.com/mozilla/tls-observatory/database"
	"github.com/mozilla/tls-observatory/logger"
)

var trustStores []TrustStore
var log = logger.GetLogger()

var TS = []string{ubuntu_TS_name, mozilla_TS_name, microsoft_TS_name, apple_TS_name, android_TS_name}

func Setup(c config.Config, DB *pg.DB) {
	ts := c.TrustStores

	db = DB

	for _, name := range TS {

		path := ""

		switch name {
		case ubuntu_TS_name:

			path = ts.UbuntuTS

		case mozilla_TS_name:

			path = ts.MozillaTS

		case microsoft_TS_name:

			path = ts.MicrosoftTS

		case apple_TS_name:
			path = ts.AppleTS

		case android_TS_name:
			path = ts.AndroidTS

		default:

			log.WithFields(logrus.Fields{
				"tsname": name,
			}).Warning("Wrong slice name entry. This should not be there")
		}

		log.WithFields(logrus.Fields{
			"tsname": name,
			"path":   path,
		}).Debug("Loading Truststore")

		// load the entire trustore into pooldata, then iterate over each PEM block
		// until all of pooldata is read
		poolData, err := ioutil.ReadFile(path)
		if err != nil {
			log.WithFields(logrus.Fields{
				"tsname": name,
				"error":  err.Error(),
			}).Warning("Failed to load truststore")
		}
		certPool := x509.NewCertPool()
		poollen := 0
		for len(poolData) > 0 {
			// read the next PEM block, ignore non CERTIFICATE entires
			var block *pem.Block
			block, poolData = pem.Decode(poolData)
			if block == nil {
				break
			}
			if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
				continue
			}
			// parse the current PEM block into a certificate, ignore failures
			cert, err := x509.ParseCertificate(block.Bytes)
			if err != nil {
				log.WithFields(logrus.Fields{
{'fraction_non_alphanumeric': 0.10810810810810811, 'fraction_numerical': 0.006817628439250061, 'mean_word_length': 3.474945533769063, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1322584', 'n_tokens_mistral': 1629, 'n_tokens_neox': 1373, 'n_words': 389}