<commit_before>package uidgid

import (
	"bufio"
	"fmt"
	"os"
	"runtime"
)

type IDMap string

const defaultUIDMap IDMap = "/proc/self/uid_map"
const defaultGIDMap IDMap = "/proc/self/gid_map"

func Supported() bool {
	return runtime.GOOS == "linux" &&
		defaultUIDMap.Supported() &&
		defaultGIDMap.Supported()
}

func MustGetMaxValidUID() int {
	return must(defaultUIDMap.MaxValid())
}

func MustGetMaxValidGID() int {
	return must(defaultGIDMap.MaxValid())
}

func (u IDMap) Supported() bool {
	_, err := os.Open(string(u))
	return !os.IsNotExist(err)
}

func (u IDMap) MaxValid() (int, error) {
	f, err := os.Open(string(u))
	if err != nil {
		return 0, err
	}

	m := 0
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		var container, host, size int
		if _, err := fmt.Sscanf(scanner.Text(), "%d %d %d", &container, &host, &size); err != nil {
			return 0, ParseError{Line: scanner.Text(), Err: err}
		}

		m = max(m, container+size-1)
	}

	return m, nil
}

func min(a, b int) int {
	if a < b {
		return a
	}

	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}

	return b
}

type ParseError struct {
	Line string
	Err  error
}

func (p ParseError) Error() string {
	return fmt.Sprintf(`%s while parsing line "%s"`, p.Err, p.Line)
}

func must(a int, err error) int {
	if err != nil {
		panic(err)
	}

	return a
}
<commit_msg>fix int overflow on 32bit platforms<commit_after>package uidgid

import (
	"bufio"
	"fmt"
	"os"
	"runtime"
)

type IDMap string

const defaultUIDMap IDMap = "/proc/self/uid_map"
const defaultGIDMap IDMap = "/proc/self/gid_map"

const maxInt = int(^uint(0) >> 1)

func Supported() bool {
	return runtime.GOOS == "linux" &&
		defaultUIDMap.Supported() &&
		defaultGIDMap.Supported()
}

func MustGetMaxValidUID() int {
	return must(defaultUIDMap.MaxValid())
}

func MustGetMaxValidGID() int {
	return must(defaultGIDMap.MaxValid())
}

func (u IDMap) Supported() bool {
	_, err := os.Open(string(u))
	return !os.IsNotExist(err)
}

func (u IDMap) MaxValid() (int, error) {
	f, err := os.Open(string(u))
	if err != nil {
		return 0, err
	}

	var m uint
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		var container, host, size uint
		if _, err := fmt.Sscanf(scanner.Text(), "%d %d %d", &container, &host, &size); err != nil {
			return 0, ParseError{Line: scanner.Text(), Err: err}
		}

		m = minUint(container+size-1, uint(maxInt))
	}

	return int(m), nil
}

func min(a, b int) int {
	if a < b {
		return a
	}

	return b
}

func minUint(a, b uint) uint {
	if a < b {
		return a
	}

	return b
}

type ParseError struct {
	Line string
	Err  error
}

func (p ParseError) Error() string {
	return fmt.Sprintf(`%s while parsing line "%s"`, p.Err, p.Line)
}

func must(a int, err error) int {
	if err != nil {
		panic(err)
	}

	return a
}

{'fraction_non_alphanumeric': 0.15158045977011494, 'fraction_numerical': 0.003951149425287357, 'mean_word_length': 3.588138385502471, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3272798', 'n_tokens_mistral': 1154, 'n_tokens_neox': 1099, 'n_words': 310}