<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logrus

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"path"
	"runtime"
	"strconv"
	"strings"
	"sync"

	"regexp"
	"sync/atomic"

	lg "github.com/Sirupsen/logrus"
	"github.com/ligato/cn-infra/logging"
	"github.com/satori/go.uuid"
)

// DefaultLoggerName is logger name of global instance of logger
const DefaultLoggerName = "defaultLogger"

var defaultLogger *Logger

// initializes global Logrus logger. Please notice, that recommended
// approach is to create a custom logger.
func init() {
	defaultLogger = NewLogger(DefaultLoggerName)
}

// DefaultLogger returns a global Logrus logger. Please notice, that recommended
// approach is to create a custom logger.
func DefaultLogger() *Logger {
	return defaultLogger
}

// Logger is wrapper of Logrus logger. In addition to Logrus functionality it
// allows to define static log fields that are added to all subsequent log entries. It also automatically
// appends file name and line where the log is coming from. In order to distinguish logs from different
// go routines a tag (number that is based on the stack address) is computed. To achieve better readability
// numeric loggers of a tag can be replaced by a string using SetTag function.
type Logger struct {
	tagmap       atomic.Value
	staticFields atomic.Value
	stdStore     atomic.Value
	std          *lg.Logger
	depth        int
	littleBuf    sync.Pool
	name         string
}

// NewLogger is a constructor creates instances of named logger.
// This constructor is called from logRegistry which is useful
// when log levels needs to be changed by management API (such as REST)
//
// Example:
//
//    logger := NewLogger("loggerXY")
//    logger.Info()
//
func NewLogger(name string) *Logger {
	logger := &Logger{
		std:   lg.New(),
		depth: 2,
		name:  name,
	}

	// init tagmap
	tags := make(map[uint64]string, 64)
	logger.tagmap.Store(tags)

	// init static fields
	statics := make(map[string]interface{})
	logger.staticFields.Store(statics)

	// init std store
	logger.stdStore.Store(logger.std)

	tf := NewTextFormatter()
	tf.TimestampFormat = "2006-01-02 15:04:05.00000"
	logger.SetFormatter(tf)

	logger.InitTag("00000000")

	logger.littleBuf.New = func() interface{} {
		buf := make([]byte, 64)
		return &buf
	}
	return logger
}

// Fields is a type for structured log entries.
type Fields map[string]interface{}

var validLoggerName = regexp.MustCompile(`^[a-zA-Z0-9.-]+$`).MatchString

func checkLoggerName(name string) error {
	if !validLoggerName(name) {
		return fmt.Errorf("logger name can contain only alphanum characters, dash and comma")
	}
	return nil
}

// NewJSONFormatter creates a new instance of JSONFormatter
func NewJSONFormatter() *lg.JSONFormatter {
	return &lg.JSONFormatter{}
}

// NewTextFormatter creates a new instance of TextFormatter
func NewTextFormatter() *lg.TextFormatter {
	return &lg.TextFormatter{}
}

// NewCustomFormatter creates a new instance of CustomFormatter
func NewCustomFormatter() *CustomFormatter {
	return &CustomFormatter{}
}

// StandardLogger returns internally used Logrus logger
func (logger *Logger) StandardLogger() *lg.Logger {
	return logger.std
}

// GetLineInfo returns the location (filename + linenumber) of the caller.
func (logger *Logger) GetLineInfo(depth int) string {
	_, f, l, ok := runtime.Caller(depth)
	if !ok {
		return ""
	}
	if f == "<autogenerated>" {
		_, f, l, ok = runtime.Caller(depth + 1)
		if !ok {
			return ""
		}
	}

	base := path.Base(f)
	dir := path.Dir(f)
	folders := strings.Split(dir, "/")
	parent := ""
	if folders != nil {
		parent = folders[len(folders)-1] + "/"
	}
	file := parent + base
	line := strconv.Itoa(l)
	return fmt.Sprintf("%s(%s)", file, line)
}

// InitTag sets the tag for the main thread.
func (logger *Logger) InitTag(tag ...string) {
	var t string
	if tag != nil || len(tag) > 0 {
		t = tag[0]
	} else {
		t = uuid.NewV4().String()[0:8]
	}
	tags := logger.tagmap.Load().(map[uint64]string)
	tags[0] = t
	logger.tagmap.Store(tags)
}

// GetTag returns the tag identifying the caller's go routine.
func (logger *Logger) GetTag() string {
	ti := logger.curGoroutineID()
	tags := logger.tagmap.Load().(map[uint64]string)
	tag, ok := tags[ti]
	if !ok {
		tag = tags[0]
	}

	return tag
}

// SetTag allows to define a string tag for the current go routine. Otherwise
// numeric identification is used.
func (logger *Logger) SetTag(tag ...string) {
	ti := logger.curGoroutineID()
	var t string
	if tag != nil || len(tag) > 0 {
		t = tag[0]
	} else {
		t = uuid.NewV4().String()[0:8]
	}
	tags := logger.tagmap.Load().(map[uint64]string)
	tags[ti] = t
	logger.tagmap.Store(tags)
}

// ClearTag removes the previously set string tag for the current go routine.
func (logger *Logger) ClearTag() {
	ti := logger.curGoroutineID()
	tags := logger.tagmap.Load().(map[uint64]string)
	delete(tags, ti)
	logger.tagmap.Store(tags)
}

// SetStaticFields sets a map of fields that will be part of the each subsequent
// log entry of the logger
func (logger *Logger) SetStaticFields(fields map[string]interface{}) {
	logger.staticFields.Store(fields)
}

// GetStaticFields returns currently set map of static fields - key-loggers pairs
// that are automatically added into log entry
func (logger *Logger) GetStaticFields() map[string]interface{} {
	return logger.staticFields.Load().(map[string]interface{})
}

// GetName return the logger name
func (logger *Logger) GetName() string {
	return logger.name
}

// SetOutput sets the standard logger output.
func (logger *Logger) SetOutput(out io.Writer) {
	std := logger.stdStore.Load().(*lg.Logger)
	std.Out = out
	logger.stdStore.Store(std)

}

// SetFormatter sets the standard logger formatter.
func (logger *Logger) SetFormatter(formatter lg.Formatter) {
	std := logger.stdStore.Load().(*lg.Logger)
	std.Formatter = formatter
	logger.stdStore.Store(std)
}

// SetLevel sets the standard logger level.
func (logger *Logger) SetLevel(level logging.LogLevel) {
	std := logger.stdStore.Load().(*lg.Logger)
	switch level {
	case logging.PanicLevel:
		std.Level = lg.PanicLevel
	case logging.FatalLevel:
		std.Level = lg.FatalLevel
	case logging.ErrorLevel:
		std.Level = lg.ErrorLevel
	case logging.WarnLevel:
		std.Level = lg.WarnLevel
	case logging.InfoLevel:
		std.Level = lg.InfoLevel
	case logging.DebugLevel:
		std.Level = lg.DebugLevel
	}
	// Store previously set level
	logger.stdStore.Store(std)
}

// GetLevel returns the standard logger level.
func (logger *Logger) GetLevel() logging.LogLevel {
	l := lg.Level(atomic.LoadUint32((*uint32)(&logger.std.Level)))
	switch l {
	case lg.PanicLevel:
		return logging.PanicLevel
	case lg.FatalLevel:
		return logging.FatalLevel
	case lg.ErrorLevel:
		return logging.ErrorLevel
	case lg.WarnLevel:
		return logging.WarnLevel
	case lg.InfoLevel:
		return logging.InfoLevel
	case lg.DebugLevel:
		return logging.DebugLevel
	default:
		return logging.DebugLevel
	}
}

// AddHook adds a hook to the standard logger hooks.
func (logger *Logger) AddHook(hook lg.Hook) {
	std := logger.stdStore.Load().(*lg.Logger)
	logger.std.Hooks.Add(hook)
	logger.stdStore.Store(std)
}

func (logger *Logger) withField(key string, value interface{}, depth ...int) *LogMsg {
	d := 1
	if depth != nil && len(depth) > 0 {
		d += depth[0]
	}
	return logger.withFields(Fields{key: value}, d)
}

// WithField creates an entry from the standard logger and adds a field to
// it. If you want multiple fields, use `WithFields`.
//
// Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal
// or Panic on the LogMsg it returns.
func (logger *Logger) WithField(key string, value interface{}) logging.LogWithLevel {
	return logger.withField(key, value, 1)
}

func (logger *Logger) withFields(fields Fields, depth ...int) *LogMsg {
	d := logger.depth
	if depth != nil && len(depth) > 0 {
		d += depth[0]
	}

	static := logger.GetStaticFields()

	f := make(lg.Fields, len(fields)+len(static))

	for k, v := range static {
		f[k] = v
	}

	for k, v := range fields {
		f[k] = v
	}

	if _, ok := f[tagKey]; !ok {
		f[tagKey] = logger.GetTag()
	}
	if _, ok := f[locKey]; !ok {
		f[locKey] = logger.GetLineInfo(d)
	}
	f[loggerKey] = logger.name

	entry := logger.std.WithFields(f)
	return &LogMsg{
		logger: logger,
		Entry:  entry,
	}
}

// WithFields creates an entry from the standard logger and adds multiple
// fields to it. This is simply a helper for `WithField`, invoking it
// once for each field.
//
// Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal
// or Panic on the LogMsg it returns.
func (logger *Logger) WithFields(fields map[string]interface{}) logging.LogWithLevel {
	return logger.withFields(Fields(fields), 1)
}

func (logger *Logger) header(depth int) *LogMsg {
	t := logger.GetTag()
	l := logger.GetLineInfo(logger.depth + depth)
	e := logger.withFields(Fields{
		tagKey: t,
		locKey: l,
	}, 2)
	return e
}

// Debug logs a message at level Debug on the standard logger.
func (logger *Logger) Debug(args ...interface{}) {
	logger.header(1).Debug(args...)
}

// Print logs a message at level Info on the standard logger.
func (logger *Logger) Print(args ...interface{}) {
	logger.std.Print(args...)
}
<commit_msg>atomic store/load of pointers<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logrus

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"path"
	"runtime"
	"strconv"
	"strings"
	"sync"

	"regexp"
	"sync/atomic"

	lg "github.com/Sirupsen/logrus"
	"github.com/ligato/cn-infra/logging"
	"github.com/satori/go.uuid"
	"unsafe"
)

// DefaultLoggerName is logger name of global instance of logger
const DefaultLoggerName = "defaultLogger"

var defaultLogger *Logger

// initializes global Logrus logger. Please notice, that recommended
// approach is to create a custom logger.
func init() {
	defaultLogger = NewLogger(DefaultLoggerName)
}

// DefaultLogger returns a global Logrus logger. Please notice, that recommended
// approach is to create a custom logger.
func DefaultLogger() *Logger {
	return defaultLogger
}

// Logger is wrapper of Logrus logger. In addition to Logrus functionality it
// allows to define static log fields that are added to all subsequent log entries. It also automatically
// appends file name and line where the log is coming from. In order to distinguish logs from different
// go routines a tag (number that is based on the stack address) is computed. To achieve better readability
// numeric value of a tag can be replaced by a string using SetTag function.
type Logger struct {
	tagmap       atomic.Value
	staticFields atomic.Value
	std          *lg.Logger
	depth        int
	littleBuf    sync.Pool
	name         string
}

// NewLogger is a constructor creates instances of named logger.
// This constructor is called from logRegistry which is useful
// when log levels needs to be changed by management API (such as REST)
//
// Example:
//
//    logger := NewLogger("loggerXY")
//    logger.Info()
//
func NewLogger(name string) *Logger {
	logger := &Logger{
		std:   lg.New(),
		depth: 2,
		name:  name,
	}

	// init tagmap
	tags := make(map[uint64]string, 64)
	logger.tagmap.Store(tags)

	// init static fields
	statics := make(map[string]interface{})
	logger.staticFields.Store(statics)

	// store std
	atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&logger.std)), unsafe.Pointer(logger.std))

	tf := NewTextFormatter()
	tf.TimestampFormat = "2006-01-02 15:04:05.00000"
	logger.SetFormatter(tf)

	logger.InitTag("00000000")

	logger.littleBuf.New = func() interface{} {
		buf := make([]byte, 64)
		return &buf
	}
	return logger
}

// Fields is a type for structured log entries.
type Fields map[string]interface{}

var validLoggerName = regexp.MustCompile(`^[a-zA-Z0-9.-]+$`).MatchString

func checkLoggerName(name string) error {
	if !validLoggerName(name) {
		return fmt.Errorf("logger name can contain only alphanum characters, dash and comma")
	}
	return nil
}

// NewJSONFormatter creates a new instance of JSONFormatter
func NewJSONFormatter() *lg.JSONFormatter {
	return &lg.JSONFormatter{}
}

// NewTextFormatter creates a new instance of TextFormatter
func NewTextFormatter() *lg.TextFormatter {
	return &lg.TextFormatter{}
}

// NewCustomFormatter creates a new instance of CustomFormatter
func NewCustomFormatter() *CustomFormatter {
	return &CustomFormatter{}
}

// StandardLogger returns internally used Logrus logger
func (logger *Logger) StandardLogger() *lg.Logger {
	return logger.std
}

// GetLineInfo returns the location (filename + linenumber) of the caller.
func (logger *Logger) GetLineInfo(depth int) string {
	_, f, l, ok := runtime.Caller(depth)
	if !ok {
		return ""
	}
	if f == "<autogenerated>" {
		_, f, l, ok = runtime.Caller(depth + 1)
		if !ok {
			return ""
		}
	}

	base := path.Base(f)
	dir := path.Dir(f)
	folders := strings.Split(dir, "/")
	parent := ""
	if folders != nil {
		parent = folders[len(folders)-1] + "/"
	}
	file := parent + base
	line := strconv.Itoa(l)
	return fmt.Sprintf("%s(%s)", file, line)
}

// InitTag sets the tag for the main thread.
func (logger *Logger) InitTag(tag ...string) {
	var t string
	if tag != nil || len(tag) > 0 {
		t = tag[0]
	} else {
		t = uuid.NewV4().String()[0:8]
	}
	tags := logger.tagmap.Load().(map[uint64]string)
	tags[0] = t
	logger.tagmap.Store(tags)
}

// GetTag returns the tag identifying the caller's go routine.
func (logger *Logger) GetTag() string {
	ti := logger.curGoroutineID()
	tags := logger.tagmap.Load().(map[uint64]string)
	tag, ok := tags[ti]
	if !ok {
		tag = tags[0]
	}

	return tag
}

// SetTag allows to define a string tag for the current go routine. Otherwise
// numeric identification is used.
func (logger *Logger) SetTag(tag ...string) {
	ti := logger.curGoroutineID()
	var t string
	if tag != nil || len(tag) > 0 {
		t = tag[0]
	} else {
		t = uuid.NewV4().String()[0:8]
	}
	tags := logger.tagmap.Load().(map[uint64]string)
	tags[ti] = t
	logger.tagmap.Store(tags)
}

// ClearTag removes the previously set string tag for the current go routine.
func (logger *Logger) ClearTag() {
	ti := logger.curGoroutineID()
	tags := logger.tagmap.Load().(map[uint64]string)
	delete(tags, ti)
	logger.tagmap.Store(tags)
}

// SetStaticFields sets a map of fields that will be part of the each subsequent
// log entry of the logger
func (logger *Logger) SetStaticFields(fields map[string]interface{}) {
	logger.staticFields.Store(fields)
}

// GetStaticFields returns currently set map of static fields - key-loggers pairs
// that are automatically added into log entry
func (logger *Logger) GetStaticFields() map[string]interface{} {
	return logger.staticFields.Load().(map[string]interface{})
}

// GetName return the logger name
func (logger *Logger) GetName() string {
	return logger.name
}

// SetOutput sets the standard logger output.
func (logger *Logger) SetOutput(out io.Writer) {
	unsafeStd := (*unsafe.Pointer)(unsafe.Pointer(&logger.std))
	stdVal := (*lg.Logger)(atomic.LoadPointer(unsafeStd))
	if stdVal != nil {
		stdVal.Out = out
		atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&logger.std)), unsafe.Pointer(logger.std))
	}
}

// SetFormatter sets the standard logger formatter.
func (logger *Logger) SetFormatter(formatter lg.Formatter) {
	unsafeStd := (*unsafe.Pointer)(unsafe.Pointer(&logger.std))
	stdVal := (*lg.Logger)(atomic.LoadPointer(unsafeStd))
	if stdVal != nil {
		stdVal.Formatter = formatter
		atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&logger.std)), unsafe.Pointer(logger.std))
	}
}

// SetLevel sets the standard logger level.
func (logger *Logger) SetLevel(level logging.LogLevel) {
	std := logger.std
	switch level {
	case logging.PanicLevel:
		std.Level = lg.PanicLevel
	case logging.FatalLevel:
		std.Level = lg.FatalLevel
	case logging.ErrorLevel:
		std.Level = lg.ErrorLevel
	case logging.WarnLevel:
		std.Level = lg.WarnLevel
	case logging.InfoLevel:
		std.Level = lg.InfoLevel
	case logging.DebugLevel:
		std.Level = lg.DebugLevel
	}
}

// GetLevel returns the standard logger level.
func (logger *Logger) GetLevel() logging.LogLevel {
	l := logger.std.Level
	switch l {
	case lg.PanicLevel:
		return logging.PanicLevel
	case lg.FatalLevel:
		return logging.FatalLevel
	case lg.ErrorLevel:
		return logging.ErrorLevel
	case lg.WarnLevel:
		return logging.WarnLevel
	case lg.InfoLevel:
		return logging.InfoLevel
	case lg.DebugLevel:
		return logging.DebugLevel
	default:
		return logging.DebugLevel
	}
}

// AddHook adds a hook to the standard logger hooks.
func (logger *Logger) AddHook(hook lg.Hook) {
	unsafeStd := (*unsafe.Pointer)(unsafe.Pointer(&logger.std))
	stdVal := (*lg.Logger)(atomic.LoadPointer(unsafeStd))
	if stdVal != nil {
		stdVal.Hooks.Add(hook)
		atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&logger.std)), unsafe.Pointer(logger.std))
	}
}

func (logger *Logger) withField(key string, value interface{}, depth ...int) *LogMsg {
	d := 1
	if depth != nil && len(depth) > 0 {
		d += depth[0]
	}
	return logger.withFields(Fields{key: value}, d)
}

// WithField creates an entry from the standard logger and adds a field to
// it. If you want multiple fields, use `WithFields`.
//
// Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal
// or Panic on the LogMsg it returns.
func (logger *Logger) WithField(key string, value interface{}) logging.LogWithLevel {
	return logger.withField(key, value, 1)
}

func (logger *Logger) withFields(fields Fields, depth ...int) *LogMsg {
	d := logger.depth
	if depth != nil && len(depth) > 0 {
		d += depth[0]
	}

	static := logger.GetStaticFields()

	f := make(lg.Fields, len(fields)+len(static))

	for k, v := range static {
		f[k] = v
	}

	for k, v := range fields {
		f[k] = v
	}

	if _, ok := f[tagKey]; !ok {
		f[tagKey] = logger.GetTag()
	}
	if _, ok := f[locKey]; !ok {
		f[locKey] = logger.GetLineInfo(d)
	}
	f[loggerKey] = logger.name

	entry := logger.std.WithFields(f)
	return &LogMsg{
		logger: logger,
		Entry:  entry,
	}
}

// WithFields creates an entry from the standard logger and adds multiple
// fields to it. This is simply a helper for `WithField`, invoking it
// once for each field.
//
// Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal
// or Panic on the LogMsg it returns.
func (logger *Logger) WithFields(fields map[string]interface{}) logging.LogWithLevel {
	return logger.withFields(Fields(fields), 1)
}

func (logger *Logger) header(depth int) *LogMsg {
	t := logger.GetTag()
	l := logger.GetLineInfo(logger.depth + depth)
	e := logger.withFields(Fields{
		tagKey: t,
		locKey: l,
	}, 2)
	return e
}

// Debug logs a message at level Debug on the standard logger.
func (logger *Logger) Debug(args ...interface{}) {
	logger.header(1).Debug(args...)
}

// Print logs a message at level Info on the standard logger.
func (logger *Logger) Print(args ...interface{}) {
	unsafeStd := (*unsafe.Pointer)(unsafe.Pointer(&logger.std))
	stdVal := (*lg.Logger)(atomic.LoadPointer(unsafeStd))
	if stdVal != nil {
		stdVal.Print(args...)
	}
}

{'fraction_non_alphanumeric': 0.12593255744553863, 'fraction_numerical': 0.007659405152690739, 'mean_word_length': 3.8874574623237725, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1844615', 'n_tokens_mistral': 6693, 'n_tokens_neox': 6413, 'n_words': 2094}