<commit_before>package core

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"pilosa/config"
	"pilosa/db"
	"pilosa/index"
	"pilosa/util"
	"reflect"
	"runtime"
	"strconv"
	"strings"

	"time"

	notify "github.com/bitly/go-notify"
	"github.com/davecgh/go-spew/spew"
	"github.com/gorilla/websocket"
	"tux21b.org/v1/gocql/uuid"
)

type WebService struct {
	service *Service
}

func NewWebService(service *Service) *WebService {
	return &WebService{service}
}

func (self *WebService) Run() {
	port_string := strconv.Itoa(config.GetInt("port_http"))
	log.Printf("Serving HTTP on port %s...\n", port_string)
	mux := http.NewServeMux()
	mux.HandleFunc("/message", self.HandleMessage)
	mux.HandleFunc("/query", self.HandleQuery)
	mux.HandleFunc("/stats", self.HandleStats)
	mux.HandleFunc("/status", self.HandleStatus)
	mux.HandleFunc("/info", self.HandleInfo)
	mux.HandleFunc("/processes", self.HandleProcesses)
	mux.HandleFunc("/listen/ws", self.HandleListenWS)
	mux.HandleFunc("/listen/stream", self.HandleListenStream)
	mux.HandleFunc("/listen", self.HandleListen)
	mux.HandleFunc("/test", self.HandleTest)
	mux.HandleFunc("/version", self.HandleVersion)
	mux.HandleFunc("/ping", self.HandlePing)
	mux.HandleFunc("/batch", self.HandleBatch)
	mux.HandleFunc("/set_bits", self.HandleSetBit)
	s := &http.Server{
		Addr:    ":" + port_string,
		Handler: mux,
	}
	s.ListenAndServe()
}

func (self *WebService) HandleMessage(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return<commit_msg>added setbit logging<commit_after>package core

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"pilosa/config"
	"pilosa/db"
	"pilosa/index"
	"pilosa/util"
	"reflect"
	"runtime"
	"strconv"
	"strings"

	"time"

	notify "github.com/bitly/go-notify"
	"github.com/davecgh/go-spew/spew"
	"github.com/gorilla/websocket"
	"tux21b.org/v1/gocql/uuid"
)

type WebService struct {
	service *Service
}

func NewWebService(service *Service) *WebService {
	return &WebService{service}
}

type RequestLogger struct {
	handler http.HandlerFunc
	logger  chan []byte
}

func NewRequestLogger(handler http.HandlerFunc, logger chan []byte) http.HandlerFunc {
	r := RequestLogger{handler, logger}
	return r.Handle
}
func (self *RequestLogger) Handle(w http.ResponseWriter, r *http.Request) {

	// Grab a dump of the incoming request
	dump, err := httputil.DumpRequest(r, true /*dump the body also*/)
	if err != nil {
		log.Println("Dump Failure", err)
	}

	self.handler(w, r)
	self.logger <- dump
}

type LogRecord struct {
	when     time.Time
	data_x64 string
}

func NewLogRecord(t time.Time, data []byte) LogRecord {
	var b bytes.Buffer
	w := gzip.NewWriter(&b)
	w.Write(data)
	w.Flush()
	w.Close()
	e := base64.StdEncoding.EncodeToString(b.Bytes())
	x := LogRecord{t, e}
	return x
}
func genFileName(id string) string {
	//bucket/YYYY/MM/DDHHMMSS.id.log
	t := time.Now()
	base := "http://pilosa.umbel.com.s3.amazonaws.com/bit_log"
	return fmt.Sprintf("%s%s.%s.log", base, t.Format("/2006/01/02/15/04-05"), id)

}
func flush(requests []LogRecord, id string) {
	dest := genFileName(id)
	w, err := util.Create(dest)
	if err != nil {
		log.Println("Error opening outfile ", dest)
		log.Println(err)
	}
	defer w.Close()

	encoder := json.NewEncoder(w)
	encoder.Encode(requests)

}
func Logger(in chan []byte, end chan bool, id string) {
	var buffer = make([]LogRecord, 2048, 2048)
	for {
		select {
		case raw := <-in:
			log.Println(string(raw))
			logRecord := NewLogRecord(time.Now(), raw)
			buffer = append(buffer, logRecord)
			if len(buffer) > 2047 {
				flush(buffer, id)
				buffer = make([]LogRecord, 2048, 2048)
			}
		case <-end:
			flush(buffer, id)
			log.Println("Shutdown Logger")
			return
		}

	}
}
func (self *WebService) Run() {
	port_string := strconv.Itoa(config.GetInt("port_http"))
	log.Printf("Serving HTTP on port %s...\n", port_string)
	logger_chan := make(chan []byte, 1024)
	end := make(chan bool)
	mux := http.NewServeMux()
	mux.HandleFunc("/message", self.HandleMessage)
	mux.HandleFunc("/query", self.HandleQuery)
	mux.HandleFunc("/stats", self.HandleStats)
	mux.HandleFunc("/status", self.HandleStatus)
	mux.HandleFunc("/info", self.HandleInfo)
	mux.HandleFunc("/processes", self.HandleProcesses)
	mux.HandleFunc("/listen/ws", self.HandleListenWS)
	mux.HandleFunc("/listen/stream", self.HandleListenStream)
	mux.HandleFunc("/listen", self.HandleListen)
	mux.HandleFunc("/test", self.HandleTest)
	mux.HandleFunc("/version", self.HandleVersion)
	mux.HandleFunc("/ping", self.HandlePing)
	mux.HandleFunc("/batch", self.HandleBatch)
	mux.HandleFunc("/set_bits", NewRequestLogger(self.HandleSetBit, logger_chan))
	s := &http.Server{
		Addr:    ":" + port_string,
		Handler: mux,
	}
	id := config.GetString("id")
	go Logger(logger_chan, end, id)
	s.ListenAndServe()
	end <- true
}

func (self *WebService) HandleMessage(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
{'fraction_non_alphanumeric': 0.1612840086122529, 'fraction_numerical': 0.009982384028185555, 'mean_word_length': 4.627753303964758, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1243558', 'n_tokens_mistral': 1991, 'n_tokens_neox': 1902, 'n_words': 374}