<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"time"

	"github.com/dogenzaka/rotator"
	"github.com/getlantern/appdir"
	"github.com/getlantern/golog"
	"github.com/getlantern/jibber_jabber"
	"github.com/getlantern/wfilter"
	"github.com/segmentio/go-loggly"
)

var (
	LogTimestampFormat = "Jan 02 15:04:05.000"
	log                = golog.LoggerFor("flashlight")
	logglyToken        string
	versionToLoggly    = fmt.Sprintf("%v (%v)", version, buildDate)
)

func configureLogging() *rotator.SizeRotator {
	logdir := appdir.Logs("Lantern")
	log.Debugf("Placing logs in %v", logdir)
	if _, err := os.Stat(logdir); err != nil {
		if os.IsNotExist(err) {
			// Create log dir
			if err := os.MkdirAll(logdir, 0755); err != nil {
				log.Fatalf("Unable to create logdir at %s: %s", logdir, err)
			}
		}
	}
	file := rotator.NewSizeRotator(filepath.Join(logdir, "lantern.log"))
	// Set log files to 1 MB
	file.RotationSize = 1 * 1024 * 1024
	// Keep up to 20 log files
	file.MaxRotation = 20

	remoteWriter := logglyErrorWriter{loggly.New(logglyToken)}
	// Loggly has its own timestamp so don't bother adding it in message,
	// moreover, golog always write each line in whole, so we need not to care about line breaks.
	errorOut := NonStopWriter(timestamped(NonStopWriter(os.Stderr, file)), remoteWriter)
	debugOut := timestamped(NonStopWriter(os.Stdout, file))
	golog.SetOutputs(errorOut, debugOut)
	return file
}

// timestamped adds a timestamp to the beginning of log lines
func timestamped(orig io.Writer) io.Writer {
	return wfilter.LinePrepender(orig, func(w io.Writer) (int, error) {
		return fmt.Fprintf(w, "%s - ", time.Now().In(time.UTC).Format(LogTimestampFormat))
	})
}

type logglyErrorWriter struct {
	l *loggly.Client
}

func (w logglyErrorWriter) Write(b []byte) (int, error) {
	return writeToLoggly(w.l, "ERROR", string(b))
}

func writeToLoggly(l *loggly.Client, level string, msg string) (int, error) {
	lang, _ := jibber_jabber.DetectLanguage()
	country, _ := jibber_jabber.DetectTerritory()
	extra := map[string]string{
		"logLevel":  level,
		"osName":    runtime.GOOS,
		"osArch":    runtime.GOARCH,
		"osVersion": "",
		"language":  lang,
		"country":   country,
		"timeZone":  "",
		"version":   versionToLoggly,
	}
	m := loggly.Message{
		"extra":   extra,
		"message": msg,
	}
	err := l.Send(m)
	if err != nil {
		return 0, err<commit_msg>report lang, country and time zone to loggly<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"time"

	"github.com/dogenzaka/rotator"
	"github.com/getlantern/appdir"
	"github.com/getlantern/golog"
	"github.com/getlantern/jibber_jabber"
	"github.com/getlantern/wfilter"
	"github.com/segmentio/go-loggly"
)

var (
	LogTimestampFormat = "Jan 02 15:04:05.000"
	log                = golog.LoggerFor("flashlight")
	logglyToken        string
	versionToLoggly    = fmt.Sprintf("%v (%v)", version, buildDate)
	lang               string
	country            string
	tz                 string
)

func init() {
	lang, _ = jibber_jabber.DetectLanguage()
	country, _ = jibber_jabber.DetectTerritory()
	tz = time.Local.String()
}

func configureLogging() *rotator.SizeRotator {
	logdir := appdir.Logs("Lantern")
	log.Debugf("Placing logs in %v", logdir)
	if _, err := os.Stat(logdir); err != nil {
		if os.IsNotExist(err) {
			// Create log dir
			if err := os.MkdirAll(logdir, 0755); err != nil {
				log.Fatalf("Unable to create logdir at %s: %s", logdir, err)
			}
		}
	}
	file := rotator.NewSizeRotator(filepath.Join(logdir, "lantern.log"))
	// Set log files to 1 MB
	file.RotationSize = 1 * 1024 * 1024
	// Keep up to 20 log files
	file.MaxRotation = 20

	remoteWriter := logglyErrorWriter{loggly.New(logglyToken)}
	// Loggly has its own timestamp so don't bother adding it in message,
	// moreover, golog always write each line in whole, so we need not to care about line breaks.
	errorOut := NonStopWriter(timestamped(NonStopWriter(os.Stderr, file)), remoteWriter)
	debugOut := timestamped(NonStopWriter(os.Stdout, file))
	golog.SetOutputs(errorOut, debugOut)
	return file
}

// timestamped adds a timestamp to the beginning of log lines
func timestamped(orig io.Writer) io.Writer {
	return wfilter.LinePrepender(orig, func(w io.Writer) (int, error) {
		return fmt.Fprintf(w, "%s - ", time.Now().In(time.UTC).Format(LogTimestampFormat))
	})
}

type logglyErrorWriter struct {
	l *loggly.Client
}

func (w logglyErrorWriter) Write(b []byte) (int, error) {
	return writeToLoggly(w.l, "ERROR", string(b))
}

func writeToLoggly(l *loggly.Client, level string, msg string) (int, error) {
	extra := map[string]string{
		"logLevel":  level,
		"osName":    runtime.GOOS,
		"osArch":    runtime.GOARCH,
		"osVersion": "",
		"language":  lang,
		"country":   country,
		"timeZone":  tz,
		"version":   versionToLoggly,
	}
	m := loggly.Message{
		"extra":   extra,
		"message": msg,
	}
	err := l.Send(m)
	if err != nil {
		return 0, err
{'fraction_non_alphanumeric': 0.1410385936552839, 'fraction_numerical': 0.01212366134572641, 'mean_word_length': 3.876847290640394, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5655609', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1806, 'n_words': 437}