<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"syscall"
)

func init() {
	log.SetFlags(0)
	log.SetOutput(os.Stderr)
}

// getting from /usr/include/syslog.h
const (
	LOG_KERN    = 0 << 3
	LOG_FACMASK = 0x03f8
)

var (
	cflag = flag.Bool("c", false, "Clear kernel buffer")
)

func isKernSyslogEntry(fac int64) bool {
	return (fac & LOG_FACMASK) == LOG_KERN
}

func main() {
	flag.Parse()

	msgbuf, err := syscall.Sysctl("kern.msgbuf")
	if err != nil {
		log.Fatalf("sysctl kern.msgbuf: %v", err)
	}

	buf := []byte(msgbuf)
	//trim leading \0 chars ( \0 == \x00 )
	for i := 0; i < len(buf); i++ {
		if buf[i] != '\x00' {
			buf = buf[i:]
			break
		}
	}

	if buf[len(buf)-1] == '\n' {
		buf = buf[0 : len(buf)-1]
	}

	lines := strings.Split(string(buf), "\n")
	for _, line := range lines {<commit_msg>dmesg update<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"syscall"
)

func init() {
	log.SetFlags(0)
	log.SetOutput(os.Stderr)
}

// getting from /usr/include/syslog.h
const (
	LOG_KERN    = 0 << 3
	LOG_FACMASK = 0x03f8
)

var (
	cflag = flag.Bool("c", false, "Clear kernel message buffer")
)

func isKernSyslogEntry(fac int64) bool {
	return (fac & LOG_FACMASK) == LOG_KERN
}

func main() {
	flag.Parse()

	msgbuf, err := syscall.Sysctl("kern.msgbuf")
	if err != nil {
		log.Fatalf("sysctl kern.msgbuf: %v", err)
	}

	buf := []byte(msgbuf)
	//trim leading \0 chars ( \0 == \x00 )
	for i := 0; i < len(buf); i++ {
		if buf[i] != '\x00' {
			buf = buf[i:]
			break
		}
	}

	if buf[len(buf)-1] == '\n' {
		buf = buf[0 : len(buf)-1]
	}

	lines := strings.Split(string(buf), "\n")
	for _, line := range lines {
{'fraction_non_alphanumeric': 0.1940385739333723, 'fraction_numerical': 0.022209234365867914, 'mean_word_length': 3.125301204819277, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1604257', 'n_tokens_mistral': 805, 'n_tokens_neox': 752, 'n_words': 172}