<commit_before>package reporter

import (
	"fmt"
	"github.com/fw42/go-hpfeeds"
	"sync"
	"time"
)

// Taken from http://blog.golang.org/pipelines
// Modified to support chan []byte instead of <-chan int
func merge(cs ...chan []byte) chan []byte {
	var wg sync.WaitGroup
	out := make(chan []byte)

	// Start an output goroutine for each input channel in cs.  output
	// copies values from c to out until c is closed, then calls wg.Done.
	output := func(c <-chan []byte) {
		for n := range c {
			out <- n
		}
		wg.Done()
	}
	wg.Add(len(cs))
	for _, c := range cs {
		go output(c)
	}

	// Start a goroutine to close out once all the output goroutines are
	// done.  This must start after the wg.Add call.
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

func Start(host string, port int, ident string, auth string, outputs []chan []byte) {

	hp := hpfeeds.NewHpfeeds(host, port, ident, auth)
	hp.Log = true
	hp.Connect()

	// Publish something on "flotest" every second
	// Channel1 is where we put the filtered JSON data
	channel1 := merge(outputs...)
	hp.Publish("beeswax.events", channel1)
	go func() {
		for {
			channel1 <- []byte("Something")
			time.Sleep(time.Second)
		}
	}()

	// Subscribe to "flotest" and print everything coming in on it
	// prints something once every second - verify with others ::
	channel2 := make(chan hpfeeds.Message)
	hp.Subscribe("beeswax.events", channel2)
	go func() {
		for foo := range channel2 {
			fmt.Println(foo.Name, string(foo.Payload))
		}
	}()

	// Wait for disconnect
	<-hp.Disconnected
}<commit_msg>Add debug output to the reporter<commit_after>package reporter

import (
	"fmt"
	"github.com/fw42/go-hpfeeds"
	"sync"
	_"time"
)

// Inspired by http://blog.golang.org/pipelines
// Modified to support chan []byte instead of <-chan int
func merge(cs ...chan []byte) (chan []byte, chan []byte) {
	var wg sync.WaitGroup
	out := make(chan []byte)
	debug := make(chan []byte)

	// Start an output goroutine for each input channel in cs.  output
	// copies values from c to out until c is closed, then calls wg.Done.
	output := func(c <-chan []byte) {
		for n := range c {
			// Make it so the channels won't block each other
			go func() { debug <- n }()
			go func() { out <- n }()
		}
		wg.Done()
	}
	wg.Add(len(cs))
	for _, c := range cs {
		go output(c)
	}

	// Start a goroutine to close out once all the output goroutines are
	// done.  This must start after the wg.Add call.
	go func() {
		wg.Wait()
		close(out)
		close(debug)
	}()
	return out, debug
}

func Start(host string, port int, ident string, auth string, outputs []chan []byte) {
	hp := hpfeeds.NewHpfeeds(host, port, ident, auth)
	hp.Log = true
	// Channel1 is where we put the filtered JSON data
	channel1, debug := merge(outputs...)

	go func() {
		for {
			message := <- debug
			fmt.Println("message: ", string(message))
		}
	}()

	fmt.Println("Waiting to connect to MHN server...")
	hp.Connect()
	fmt.Println("Successfully connected to MHN server!")


	// Publish something on "beeswax.events" every second
	hp.Publish("beeswax.events", channel1)
	// Subscribe to "beeswax.events" and print everything coming in on it
	// prints something once every second - verify with others ::
	channel2 := make(chan hpfeeds.Message)
	hp.Subscribe("beeswax.events", channel2)
	go func() {
		for foo := range channel2 {
			fmt.Println(foo.Name, string(foo.Payload))
		}
	}()

	// Wait for disconnect
	<-hp.Disconnected
}
{'fraction_non_alphanumeric': 0.13960944331098804, 'fraction_numerical': 0.004954823666569514, 'mean_word_length': 3.2318125770653516, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5415710', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1188, 'n_words': 398}