<commit_before>package main

import (
	"encoding/json"
	std "github.com/balzaczyy/golucene/analysis/standard"
	_ "github.com/balzaczyy/golucene/core/codec/lucene49"
	"github.com/balzaczyy/golucene/core/index"
	"github.com/balzaczyy/golucene/core/search"
	"github.com/balzaczyy/golucene/core/store"
	"github.com/balzaczyy/golucene/core/util"
	qp "github.com/balzaczyy/golucene/queryparser/classic"
	"github.com/op/go-logging"
	"net/http"
	"os"
	"path"
	"strings"
)

var log = logging.MustGetLogger("hamlet")

func init() {
	format := logging.MustStringFormatter(
		"%{color}%{time:15:04:05.000000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}")
	backend := logging.NewLogBackend(os.Stdout, "", 0)
	logging.SetBackend(logging.NewBackendFormatter(backend, format))
	log.Info("Logger is configured.")
}

func main() {

	util.SetDefaultInfoStream(util.NewPrintStreamInfoStream(os.Stdout))
	index.DefaultSimilarity = func() index.Similarity {
		return search.NewDefaultSimilarity()
	}
	directory, err := store.OpenFSDirectory("index")
	if err != nil {
		log.Critical("Failed to open directory: %v", err)
		return
	}
	defer directory.Close()
	reader, err := index.OpenDirectoryReader(directory)
	if err != nil {
		log.Critical("Failed to open writer: %v", err)
		return
	}
	defer reader.Close()
	analyzer := std.NewStandardAnalyzer()
	qParser := qp.NewQueryParser(util.VERSION_49, "text", analyzer)
	ss := search.NewIndexSearcher(reader)

	port := os.Getenv("VCAP_APP_PORT")
	if port == "" {
		port = "8081"
	}
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		file := r.URL.Path
		if path.Ext(file) == "" {<commit_msg>save log to firebase<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/balzaczyy/firebase"
	"github.com/balzaczyy/go-logging"
	std "github.com/balzaczyy/golucene/analysis/standard"
	_ "github.com/balzaczyy/golucene/core/codec/lucene410"
	"github.com/balzaczyy/golucene/core/index"
	"github.com/balzaczyy/golucene/core/search"
	"github.com/balzaczyy/golucene/core/store"
	"github.com/balzaczyy/golucene/core/util"
	qp "github.com/balzaczyy/golucene/queryparser/classic"
	"net/http"
	"os"
	"path"
	"strings"
)

var SILENT = map[string]string{
	"print": "silent",
}

type FirebaseLogger struct {
	queue  chan []byte
	closer chan chan error
}

func newLogger(root string) *FirebaseLogger {
	api := new(firebase.Client)
	api.Init(root, "", nil)
	queue := make(chan []byte, 1000)
	closer := make(chan chan error)
	go func() {
		isRunning := true
		for isRunning {
			select {
			case reply := <-closer:
				isRunning = false
				reply <- nil
			case b := <-queue:
				_, err := api.Push(string(b), SILENT)
				if err != nil {
					fmt.Println(err)
					fmt.Println(string(b))
				}
			}
		}
	}()
	return &FirebaseLogger{
		queue:  queue,
		closer: closer,
	}
}

func (logger *FirebaseLogger) Write(b []byte) (int, error) {
	logger.queue <- b
	return len(b), nil
}

func (logger *FirebaseLogger) Close() error {
	ok := make(chan error)
	logger.closer <- ok
	return <-ok
}

var log = logging.MustGetLogger("hamlet")

func main() {
	// setup logger
	logger := newLogger(
		"https://shining-inferno-3740.firebaseio.com/log/hamlet",
	)
	defer logger.Close()

	logging.SetBackend(logging.NewBackendFormatter(
		logging.NewLogBackend(logger, "", 0),
		logging.MustStringFormatter(
			"%{color}%{time:15:04:05.000000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}",
		),
	))

	// setup index and searcher
	util.SetDefaultInfoStream(util.NewPrintStreamInfoStream(os.Stdout))
	index.DefaultSimilarity = func() index.Similarity {
		return search.NewDefaultSimilarity()
	}
	directory, err := store.OpenFSDirectory("index")
	if err != nil {
		log.Critical("Failed to open directory: %v", err)
		return
	}
	defer directory.Close()
	reader, err := index.OpenDirectoryReader(directory)
	if err != nil {
		log.Critical("Failed to open writer: %v", err)
		return
	}
	defer reader.Close()
	analyzer := std.NewStandardAnalyzer()
	qParser := qp.NewQueryParser(util.VERSION_49, "text", analyzer)
	ss := search.NewIndexSearcher(reader)

	// setup web service
	port := os.Getenv("VCAP_APP_PORT")
	if port == "" {
		port = "8081"
	}
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		file := r.URL.Path
		if path.Ext(file) == "" {
{'fraction_non_alphanumeric': 0.15893271461716937, 'fraction_numerical': 0.013225058004640371, 'mean_word_length': 4.450063211125158, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '147775', 'n_tokens_mistral': 1778, 'n_tokens_neox': 1631, 'n_words': 297}