<commit_before>// Use "go run processor.go" to run this example program.

package main

import (
	"crypto/ed25519"
	"crypto/tls"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/ninchat/ninchat-go/webhook"
)

// HTTPS server program.
func main() {
	server := http.Server{
		Addr: ":443",
		TLSConfig: &tls.Config{
			MinVersion: tls.VersionTLS12,
		},
	}

	var (
		cert    = "cert.pem"
		certKey = "certkey.pem"
	)

	flag.StringVar(&server.Addr, "addr", server.Addr, "Listening [host]:port")
	flag.StringVar(&cert, "cert", cert, "TLS certificate filename")
	flag.StringVar(&certKey, "certkey", certKey, "TLS certificate's private key filename")
	flag.Parse()

	// Specify additional public keys on command-line.
	for _, s := range flag.Args() {
		parts := strings.SplitN(s, "=", 2)
		if len(parts) != 2 {
			os.Exit(2)
		}
		ninchatPublicKeys[parts[0]] = parts[1]
	}

	http.HandleFunc("/ninchat-webhook", handleNinchatWebhook)

	if cert == "" && certKey == "" {
		log.Fatal(server.ListenAndServe())
	} else {
		log.Fatal(server.ListenAndServeTLS(cert, certKey))
	}
}

// Add new key here when one is announced.  Remove the old one after it has
// been phased out.
var ninchatPublicKeys = map[string]string{
	"ninchat.com/ed25519-2019-04": "fyMJMmsTPDkp4YYxlcQcWrIwTvalCgfHMW370OE/y2I",
}

// Mitigate replay attacks by remembering X-Ninchat-Signatures until their
// expiration.  This is an optional security measure.  To be effective, the
// information should be shared among all nodes of a distributed system.
var (
	seenLock       sync.Mutex
	seenSignatures = map[string]int64{}
)

func rememberSignature(s string, exp int64) (duplicate bool) {
	seenLock.Lock()
	defer seenLock.Unlock()

	_, duplicate = seenSignatures[s]
	if !duplicate {
		seenSignatures[s] = exp
	}
	return
}

// A naive background job for removing expired signatures.
func init() {
	go func() {
		for {
			time.Sleep(time.Minute)

			seenLock.Lock()

			var expired []string

			for sig, exp := range seenSignatures {
				if exp < time.Now().Unix() {
					expired = append(expired, sig)
				}
			}

			for _, sig := range expired {
				delete(seenSignatures, sig)
			}

			seenLock.Unlock()
		}
	}()
}

// handleNinchatWebhook processes a HTTP request.
func handleNinchatWebhook(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	if !strings.HasPrefix(r.Header.Get("User-Agent"), "ninchat-webhook/") {
		w.WriteHeader(http.StatusForbidden)
		return
	}

	if r.Header.Get("Content-Type") != "application/json; charset=utf-8" {
		w.WriteHeader(http.StatusUnsupportedMediaType)
		return
	}

	if r.ContentLength < 0 {
		w.WriteHeader(http.StatusLengthRequired)
		return
	}
	if r.ContentLength > 256*1024*1024 { // Something huge...
		w.WriteHeader(http.StatusRequestEntityTooLarge) // Let Ninchat know.
		return
	}

	content, err := ioutil.ReadAll(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	var document webhook.Webhook
	if json.Unmarshal(content, &document) != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if document.Aud != "realm:abcd1234" { // This Ninchat resource id must be yours.
		w.WriteHeader(http.StatusForbidden)
		return
	}

	if document.Exp < time.Now().Unix() { // Expired?
		w.WriteHeader(http.StatusForbidden)
		return
	}

	publicKeyString, found := ninchatPublicKeys[document.Kid]
	if !found {
		w.WriteHeader(http.StatusForbidden)
		return
	}
	publicKey, err := base64.RawURLEncoding.DecodeString(publicKeyString)
	if err != nil {
		panic(err)
	}

	signatureString := r.Header.Get("X-Ninchat-Signature")
	signature, err := hex.DecodeString(signatureString)<commit_msg>webhook/example: fix public key base64 decoding<commit_after>// Use "go run processor.go" to run this example program.

package main

import (
	"crypto/ed25519"
	"crypto/tls"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/ninchat/ninchat-go/webhook"
)

// HTTPS server program.
func main() {
	server := http.Server{
		Addr: ":443",
		TLSConfig: &tls.Config{
			MinVersion: tls.VersionTLS12,
		},
	}

	var (
		cert    = "cert.pem"
		certKey = "certkey.pem"
	)

	flag.StringVar(&server.Addr, "addr", server.Addr, "Listening [host]:port")
	flag.StringVar(&cert, "cert", cert, "TLS certificate filename")
	flag.StringVar(&certKey, "certkey", certKey, "TLS certificate's private key filename")
	flag.Parse()

	// Specify additional public keys on command-line.
	for _, s := range flag.Args() {
		parts := strings.SplitN(s, "=", 2)
		if len(parts) != 2 {
			os.Exit(2)
		}
		ninchatPublicKeys[parts[0]] = parts[1]
	}

	http.HandleFunc("/ninchat-webhook", handleNinchatWebhook)

	if cert == "" && certKey == "" {
		log.Fatal(server.ListenAndServe())
	} else {
		log.Fatal(server.ListenAndServeTLS(cert, certKey))
	}
}

// Add new key here when one is announced.  Remove the old one after it has
// been phased out.
var ninchatPublicKeys = map[string]string{
	"ninchat.com/ed25519-2019-04": "<KEY>",
}

// Mitigate replay attacks by remembering X-Ninchat-Signatures until their
// expiration.  This is an optional security measure.  To be effective, the
// information should be shared among all nodes of a distributed system.
var (
	seenLock       sync.Mutex
	seenSignatures = map[string]int64{}
)

func rememberSignature(s string, exp int64) (duplicate bool) {
	seenLock.Lock()
	defer seenLock.Unlock()

	_, duplicate = seenSignatures[s]
	if !duplicate {
		seenSignatures[s] = exp
	}
	return
}

// A naive background job for removing expired signatures.
func init() {
	go func() {
		for {
			time.Sleep(time.Minute)

			seenLock.Lock()

			var expired []string

			for sig, exp := range seenSignatures {
				if exp < time.Now().Unix() {
					expired = append(expired, sig)
				}
			}

			for _, sig := range expired {
				delete(seenSignatures, sig)
			}

			seenLock.Unlock()
		}
	}()
}

// handleNinchatWebhook processes a HTTP request.
func handleNinchatWebhook(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	if !strings.HasPrefix(r.Header.Get("User-Agent"), "ninchat-webhook/") {
		w.WriteHeader(http.StatusForbidden)
		return
	}

	if r.Header.Get("Content-Type") != "application/json; charset=utf-8" {
		w.WriteHeader(http.StatusUnsupportedMediaType)
		return
	}

	if r.ContentLength < 0 {
		w.WriteHeader(http.StatusLengthRequired)
		return
	}
	if r.ContentLength > 256*1024*1024 { // Something huge...
		w.WriteHeader(http.StatusRequestEntityTooLarge) // Let Ninchat know.
		return
	}

	content, err := ioutil.ReadAll(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	var document webhook.Webhook
	if json.Unmarshal(content, &document) != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if document.Aud != "realm:abcd1234" { // This Ninchat resource id must be yours.
		w.WriteHeader(http.StatusForbidden)
		return
	}

	if document.Exp < time.Now().Unix() { // Expired?
		w.WriteHeader(http.StatusForbidden)
		return
	}

	publicKeyString, found := ninchatPublicKeys[document.Kid]
	if !found {
		w.WriteHeader(http.StatusForbidden)
		return
	}
	publicKey, err := base64.RawStdEncoding.DecodeString(publicKeyString)
	if err != nil {
		panic(err)
	}

	signatureString := r.Header.Get("X-Ninchat-Signature")
	signature, err := hex.DecodeString(signatureString)
{'fraction_non_alphanumeric': 0.1361595351294242, 'fraction_numerical': 0.01439513998943476, 'mean_word_length': 3.8235668789808916, 'pattern_counts': {'":': 4, '<': 10, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7221868', 'n_tokens_mistral': 2959, 'n_tokens_neox': 2703, 'n_words': 604}