<commit_before>package utee

import (
	"crypto/md5"
	"crypto/x509"
	"encoding/hex"
	"encoding/pem"
	"fmt"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"io"
	"math/rand"
	"net/http"
	"net/smtp"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

var (
	//passwd md5
	PlainMd5 = Md5Str("")
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

func Md5Str(salt string) func(string) string {
	return func(s string) string {
		h := md5.New()
		io.WriteString(h, s)
		io.WriteString(h, salt)
		return hex.EncodeToString(h.Sum(nil))
	}
}

func Chk(err error) {
	if err != nil {
		panic(err)
	}
}

//truncate string
func Truncate(s string, n int) string {
	length := utf8.RuneCountInString(s)
	if length <= n || n < 0 {
		return ""
	}

	l := []rune{}
	for _, r := range s {
		l = append(l, r)
	}

	l = l[:(length - n)]
	return string(l)
}

func Tick(t ...time.Time) int64 {
	if len(t) == 0 {
		return time.Now().UnixNano() / 1e6
	} else {
		return t[0].UnixNano() / 1e6
	}
}

func TickSec() int64 {
	return time.Now().Unix()
}

func TickHour() int64 {
	return time.Now().Unix() / 3600 * 3600
}

func Contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func ContainsInf(s []interface{}, e interface{}) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func ContainsObjectId(s []bson.ObjectId, e string) bool {
	for _, a := range s {
		if a.Hex() == e {
			return true
		}
	}
	return false
}

func ToTimeMillis(fmt string, timeStr string) (int64, error) {
	data, err := time.Parse(fmt, timeStr)
	if err != nil {
		return 0, err
	}
	return data.UnixNano() / 1e6, nil
}

func Md5(b []byte) []byte {
	h := md5.New()
	h.Write(b)
	return h.Sum(nil)
}

// parse form into J
func F2j(r *http.Request) J {
	r.ParseForm()
	j := J{}
	for k, v := range r.Form {
		if len(v) == 1 {
			if len(v[0]) > 0 {
				j[k] = v[0]
			}
		} else {
			j[k] = v
		}
	}
	return j
}

func Fint64(s interface{}, dft ...int64) int64 {
	var i int64
	var err error
	if s == nil {
		if len(dft) > 0 {
			return dft[0]
		} else {
			s = "0"
		}
	}
	i, err = strconv.ParseInt(s.(string), 10, 64)
	if err != nil && len(dft) > 0 {
		i = dft[0]
	}
	return i
}

func Fint(s interface{}, dft ...int64) int {
	if len(dft) > 0 {
		return int(Fint64(s, dft[0]))
	} else {
		return int(Fint64(s))
	}
}

func DeleteMap(m map[string]interface{}, ks ...string) {
	for _, v := range ks {
		delete(m, v)
	}
}

func IsPemExpire(b []byte) bool {
	block, _ := pem.Decode(b)
	if block == nil {
		fmt.Println("failed to parse certificate PEM")
		return true
	}
	cert, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		fmt.Println("failed to parse certificate: ", err)
		return true
	}
	return cert.NotAfter.Before(time.Now())
}

func NotFound(err error) bool {
	return err == mgo.ErrNotFound
}<commit_msg>Update ut.go<commit_after>package utee

import (
	"crypto/md5"
	"crypto/x509"
	"encoding/hex"
	"encoding/pem"
	"fmt"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"io"
	"math/rand"
	"net/http"
	"net/smtp"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

var (
	//passwd md5
	PlainMd5 = Md5Str("")
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

func Md5Str(salt string) func(string) string {
	return func(s string) string {
		h := md5.New()
		io.WriteString(h, s)
		io.WriteString(h, salt)
		return hex.EncodeToString(h.Sum(nil))
	}
}

func Chk(err error) {
	if err != nil {
		panic(err)
	}
}

//truncate string
func Truncate(s string, n int) string {
	length := utf8.RuneCountInString(s)
	if length <= n || n < 0 {
		return ""
	}

	l := []rune{}
	for _, r := range s {
		l = append(l, r)
	}

	l = l[:(length - n)]
	return string(l)
}

func Tick(t ...time.Time) int64 {
	if len(t) == 0 {
		return time.Now().UnixNano() / 1e6
	} else {
		return t[0].UnixNano() / 1e6
	}
}

func TickSec() int64 {
	return time.Now().Unix()
}

func TickHour() int64 {
	return time.Now().Unix() / 3600 * 3600
}

func Contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func ContainsInf(s []interface{}, e interface{}) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func ContainsObjectId(s []bson.ObjectId, e string) bool {
	for _, a := range s {
		if a.Hex() == e {
			return true
		}
	}
	return false
}

func ToTimeMillis(fmt string, timeStr string) (int64, error) {
	data, err := time.Parse(fmt, timeStr)
	if err != nil {
		return 0, err
	}
	return data.UnixNano() / 1e6, nil
}

func Md5(b []byte) []byte {
	h := md5.New()
	h.Write(b)
	return h.Sum(nil)
}

// parse form into J
func F2j(r *http.Request) J {
	r.ParseForm()
	j := J{}
	for k, v := range r.Form {
		if len(v) == 1 {
			if len(v[0]) > 0 {
				j[k] = v[0]
			}
		} else {
			j[k] = v
		}
	}
	return j
}

func Fint64(s interface{}, dft ...int64) int64 {
	var i int64
	var err error
	if s == nil {
		if len(dft) > 0 {
			return dft[0]
		} else {
			s = "0"
		}
	}
	i, err = strconv.ParseInt(s.(string), 10, 64)
	if err != nil && len(dft) > 0 {
		i = dft[0]
	}
	return i
}

func Fint(s interface{}, dft ...int64) int {
	if len(dft) > 0 {
		return int(Fint64(s, dft[0]))
	} else {
		return int(Fint64(s))
	}
}

func DeleteMap(m map[string]interface{}, ks ...string) {
	for _, v := range ks {
		delete(m, v)
	}
}

func IsPemExpire(b []byte) (bool, error) {
	block, _ := pem.Decode(b)
	if block == nil {
		return false, errors.New("failed to parse certificate PEM")
	}
	cert, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		return false, err
	}
	return cert.NotAfter.Before(time.Now()), nil
}

func NotFound(err error) bool {
	return err == mgo.ErrNotFound
}
{'fraction_non_alphanumeric': 0.17729996475149806, 'fraction_numerical': 0.02608389143461403, 'mean_word_length': 2.9003436426116838, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1800319', 'n_tokens_mistral': 2700, 'n_tokens_neox': 2514, 'n_words': 635}