<commit_before>package blocks

import (
	"bytes"
	"encoding/json"
	"log"
	"net/http"
	"strings"
)

func PostValue(b *Block) {

	type KeyMapping struct {
		MsgKey   string
		QueryKey string
	}

	type postRule struct {
		Keymapping []KeyMapping
		Endpoint   string
	}

	var rule *postRule
	client := &http.Client{}

	// TODO check the endpoint for happiness
	for {
		select {
		case msg := <-b.AddChan:
			updateOutChans(msg, b)
		case <-b.QuitChan:
			quit(b)
			return
		case msg := <-b.Routes["get_rule"]:
			if rule == nil {
				marshal(msg, &postRule{Keymapping:[]KeyMapping{KeyMapping{}}})
			} else {
				marshal(msg, rule)
			}
		case msg := <-b.Routes["set_rule"]:
			if rule == nil {
				rule = &postRule{}
			}
			unmarshal(msg, rule)

		case msg := <-b.InChan:
			if rule == nil {
				break
			}

			body := make(map[string]interface{})
			for _, keymap := range rule.Keymapping {
				keys := strings.Split(keymap.MsgKey, ".")
				value, err := Get(msg, keys...)
				if err != nil {
					log.Println(err.Error())
				} else {
					Set(body, keymap.QueryKey, value)
				}
			}

			// TODO maybe check the response ?
			postBody, err := json.Marshal(body)
			if err != nil {
				log.Fatal(err.Error())
			}

			// TODO the content-type here is heavily borked but we're using a hack
			/*resp, err := http.Post(rule.Endpoint, "application/x-www-form-urlencoded", bytes.NewReader(postBody))
			if err != nil {
				log.Println(err.Error())
				break
			}
			defer resp.Body.Close()*/

			req, err := http.NewRequest("POST", rule.Endpoint, bytes.NewReader(postBody))
			
			req.Close()

			if err != nil {
			    log.Println(err)
			    break
			}
			
			resp, err := client.Do(req)
			defer resp.Body.Close()

			if err != nil {
			    log.Println(err)
			    break
			}

			bodyBuf := &bytes.Buffer{}
			_, err = bodyBuf.ReadFrom(resp.Body)
		}
	}
}
<commit_msg>added content type to header<commit_after>package blocks

import (
	"bytes"
	"encoding/json"
	"log"
	"net/http"
	"strings"
)

func PostValue(b *Block) {

	type KeyMapping struct {
		MsgKey   string
		QueryKey string
	}

	type postRule struct {
		Keymapping []KeyMapping
		Endpoint   string
	}

	var rule *postRule
	client := &http.Client{}

	// TODO check the endpoint for happiness
	for {
		select {
		case msg := <-b.AddChan:
			updateOutChans(msg, b)
		case <-b.QuitChan:
			quit(b)
			return
		case msg := <-b.Routes["get_rule"]:
			if rule == nil {
				marshal(msg, &postRule{Keymapping:[]KeyMapping{KeyMapping{}}})
			} else {
				marshal(msg, rule)
			}
		case msg := <-b.Routes["set_rule"]:
			if rule == nil {
				rule = &postRule{}
			}
			unmarshal(msg, rule)

		case msg := <-b.InChan:
			if rule == nil {
				break
			}

			body := make(map[string]interface{})
			for _, keymap := range rule.Keymapping {
				keys := strings.Split(keymap.MsgKey, ".")
				value, err := Get(msg, keys...)
				if err != nil {
					log.Println(err.Error())
				} else {
					Set(body, keymap.QueryKey, value)
				}
			}

			// TODO maybe check the response ?
			postBody, err := json.Marshal(body)
			if err != nil {
				log.Fatal(err.Error())
			}

			// TODO the content-type here is heavily borked but we're using a hack
			req, err := http.NewRequest("POST", rule.Endpoint, bytes.NewReader(postBody))
			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
			req.Close = true

			if err != nil {
			    log.Println(err)
			    break
			}
			
			resp, err := client.Do(req)
			defer resp.Body.Close()

			if err != nil {
			    log.Println(err)
			    break
			}

			bodyBuf := &bytes.Buffer{}
			_, err = bodyBuf.ReadFrom(resp.Body)
		}
	}
}

{'fraction_non_alphanumeric': 0.14877173613027878, 'fraction_numerical': 0.0, 'mean_word_length': 2.5634218289085546, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5283788', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1283, 'n_words': 307}