<commit_before>// +build !windows

package runewidth

import (
	"os"
	"regexp"
	"strings"
)

var reLoc = regexp.MustCompile(`^[a-z][a-z][a-z]?(?:_[A-Z][A-Z])?\.(.+)`)

func IsEastAsian() bool {
	locale := os.Getenv("LC_CTYPE")
	if locale == "" {
		locale = os.Getenv("LANG")
	}

	// ignore C locale
	if locale == "POSIX" || locale == "C" {
		return false
	}
	if len(locale) > 1 && locale[0] == 'C' && (locale[1] == '.' || locale[1] == '-') {
		return false
	}

	charset := strings.ToLower(locale)
	r := reLoc.FindStringSubmatch(locale)
	if len(r) == 2 {
		charset = strings.ToLower(r[1])
	}

	if strings.HasSuffix(charset, "@cjk_narrow") {
		return false
	}

	pos := strings.IndexByte(charset, '@')
	if pos >= 0 {
		charset = charset[:pos]
	}

	mbc_max := 1
	switch charset {
	case "utf-8", "utf8":
		mbc_max = 6
	case "jis":
		mbc_max = 8
	case "eucjp":
		mbc_max = 3
	case "euckr", "euccn":
		mbc_max = 2
	case "sjis", "cp932", "cp51932", "cp936", "cp949", "cp950":
		mbc_max = 2
	case "big5":
		mbc_max = 2
	case "gbk", "gb2312":
		mbc_max = 2
	}

	if mbc_max > 1 && (charset[0] != 'u' ||
			strings.HasPrefix(locale, "ja") ||
			strings.HasPrefix(locale, "ko") ||
			strings.HasPrefix(locale, "zn")) {
		return true
	}
	return false
}<commit_msg>s/zn/zh/<commit_after>// +build !windows

package runewidth

import (
	"os"
	"regexp"
	"strings"
)

var reLoc = regexp.MustCompile(`^[a-z][a-z][a-z]?(?:_[A-Z][A-Z])?\.(.+)`)

func IsEastAsian() bool {
	locale := os.Getenv("LC_CTYPE")
	if locale == "" {
		locale = os.Getenv("LANG")
	}

	// ignore C locale
	if locale == "POSIX" || locale == "C" {
		return false
	}
	if len(locale) > 1 && locale[0] == 'C' && (locale[1] == '.' || locale[1] == '-') {
		return false
	}

	charset := strings.ToLower(locale)
	r := reLoc.FindStringSubmatch(locale)
	if len(r) == 2 {
		charset = strings.ToLower(r[1])
	}

	if strings.HasSuffix(charset, "@cjk_narrow") {
		return false
	}

	pos := strings.IndexByte(charset, '@')
	if pos >= 0 {
		charset = charset[:pos]
	}

	mbc_max := 1
	switch charset {
	case "utf-8", "utf8":
		mbc_max = 6
	case "jis":
		mbc_max = 8
	case "eucjp":
		mbc_max = 3
	case "euckr", "euccn":
		mbc_max = 2
	case "sjis", "cp932", "cp51932", "cp936", "cp949", "cp950":
		mbc_max = 2
	case "big5":
		mbc_max = 2
	case "gbk", "gb2312":
		mbc_max = 2
	}

	if mbc_max > 1 && (charset[0] != 'u' ||
			strings.HasPrefix(locale, "ja") ||
			strings.HasPrefix(locale, "ko") ||
			strings.HasPrefix(locale, "zh")) {
		return true
	}
	return false
}
{'fraction_non_alphanumeric': 0.21042084168336672, 'fraction_numerical': 0.03286573146292585, 'mean_word_length': 3.409893992932862, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '188302', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1086, 'n_words': 232}