<commit_before>// Copyright 2014 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package brimtext contains tools for working with text. Probably the most
// complex of these tools is Align, which allows for formatting "pretty
// tables".
package brimtext

import (
	"bytes"
	"strconv"
	"strings"
)

// OrdinalSuffix returns "st", "nd", "rd", etc. for the number given (1st, 2nd,
// 3rd, etc.).
func OrdinalSuffix(number int) string {
	if (number/10)%10 == 1 || number%10 > 3 {
		return "th"
	} else if number%10 == 1 {
		return "st"
	} else if number%10 == 2 {
		return "nd"
	} else if number%10 == 3 {
		return "rd"
	}
	return "th"
}

// ThousandsSep returns the number formatted using the separator at each
// thousands position, such as ThousandsSep(1234567, ",") giving 1,234,567.
func ThousandsSep(v int64, sep string) string {
	s := strconv.FormatInt(v, 10)
	for i := len(s) - 3; i > 0; i -= 3 {
		s = s[:i] + "," + s[i:]
	}
	return s
}

type humanSize struct {
	d int64
	s string
}

var humanSizes = []humanSize{
	humanSize{int64(1024), "K"},
	humanSize{int64(1024) << 10, "M"},
	humanSize{int64(1024) << 20, "G"},
	humanSize{int64(1024) << 30, "T"},
	humanSize{int64(1024) << 40, "P"},
	humanSize{int64(1024) << 50, "E"},
}

// Returns a more readable size format, such as HumanSize(1234567, "") giving
// "1M". For values less than 1K, it is common that no suffix letter should be
// added; but the appendBytes parameter is provided in case clarity is needed.
func HumanSize(b int64, appendBytes string) string {
	if b < 1024 {
		v := strconv.FormatInt(b, 10)
		if appendBytes != "" {
			return v + appendBytes
		}
		return v
	}
	c := b
	s := appendBytes
	for _, h := range humanSizes {
		c = b / h.d
		r := b % h.d
		if r >= h.d/2 {
			c++
		}
		if c < 1024 {
			s = h.s
			break
		}
	}
	return strconv.FormatInt(c, 10) + s
}

type Alignment int

const (
	Left Alignment = iota
	Right
	Center
)

type AlignOptions struct {
	// Widths indicate the desired widths of each column. If nil or if a value
	// is 0, no rewrapping will be done.
	Widths     []int
	Alignments []Alignment
	// FirstDR etc. control what is output for situations with a prepended
	// display row, First row output with Down and Right connections, etc.
	FirstDR       string
	FirstLR       string
	FirstFirstDLR string
	FirstDLR      string
	FirstDL       string
	// RowFirstUD etc. control situations for each data row output.
	RowFirstUD  string
	RowSecondUD string
	RowUD       string
	RowLastUD   string
	// LeaveTrailingWhitespace should be set true if the last cell of data row
	// needs spaces to fill to the end (usually needed when setting RawLastUD).
	LeaveTrailingWhitespace bool
	// FirstNilFirstUDR etc. control situations when the first nil data row is
	// encountered. Can be used to separate the header from the rest of the
	// rows.
	FirstNilFirstUDR  string
	FirstNilLR        string
	FirstNilFirstUDLR string
	FirstNilUDLR      string
	FirstNilLastUDL   string
	// NilFirstUDR etc. control situations when the second and subsequent nil
	// data rows are encountered. Can be used to separate rows from each other.
	NilFirstUDR  string
	NilLR        string
	NilFirstUDLR string
	NilUDLR      string
	NilLastUDL   string
	// LastUR etc. control what is output for situations with an appended
	// display row.
	LastUR       string
	LastLR       string
	LastFirstULR string
	LastULR      string
	LastUL       string
	// NilBetweenEveryRow will add a nil data row between all rows; use to emit
	// FirstNil* and Nil* row separators.
	NilBetweenEveryRow bool
}

// NewDefaultAlignOptions gives:
//
//  &AlignOptions{RowSecondUD: " ", RowUD: " "}
//
// Which will format tables like:
//
//           Bob         Sue    John
//  Hometown San Antonio Austin New York
//  Mother   Bessie      Mary   Sarah
//  Father   Rick        Dan    Mike
func NewDefaultAlignOptions() *AlignOptions {
	return &AlignOptions{RowSecondUD: " ", RowUD: " "}
}

// NewSimpleAlignOptions gives:
//
//  return &AlignOptions{
//      FirstDR:                 "+-",
//      FirstLR:                 "-",
//      FirstFirstDLR:           "-+-",
//      FirstDLR:                "-+-",
//      FirstDL:                 "-+",
//      RowFirstUD:              "| ",
//      RowSecondUD:             " | ",
//      RowUD:                   " | ",
//      RowLastUD:               " |",
//      LeaveTrailingWhitespace: true,
//      FirstNilFirstUDR:        "+-",
//      FirstNilLR:              "-",
//      FirstNilFirstUDLR:       "-+-",
//      FirstNilUDLR:            "-+-",
//      FirstNilLastUDL:         "-+",
//      LastUR:                  "+-",
//      LastLR:                  "-",
//      LastFirstULR:            "-+-",
//      LastULR:                 "-+-",
//      LastUL:                  "-+",
//  }
//
// Which will format tables like:
//
// +----------+-------------+--------+----------+
// |          | Bob         | Sue    | John     |
// +----------+-------------+--------+----------+
// | Hometown | San Antonio | Austin | New York |
// | Mother   | Bessie      | Mary   | Sarah    |
// | Father   | Rick        | Dan    | Mike     |
// +----------+-------------+--------+----------+
func NewSimpleAlignOptions() *AlignOptions {
	return &AlignOptions{
		FirstDR:                 "+-",
		FirstLR:                 "-",
		FirstFirstDLR:           "-+-",
		FirstDLR:                "-+-",
		FirstDL:                 "-+",
		RowFirstUD:              "| ",
		RowSecondUD:             " | ",
		RowUD:                   " | ",
		RowLastUD:               " |",
		LeaveTrailingWhitespace: true,
		FirstNilFirstUDR:        "+-",
		FirstNilLR:              "-",
		FirstNilFirstUDLR:       "-+-",
		FirstNilUDLR:            "-+-",
		FirstNilLastUDL:         "-+",
		LastUR:                  "+-",
		LastLR:                  "-",
		LastFirstULR:            "-+-",
		LastULR:                 "-+-",
		LastUL:                  "-+",
	}
}
<commit_msg>Fixed doc format issue.<commit_after>// Copyright 2014 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package brimtext contains tools for working with text. Probably the most
// complex of these tools is Align, which allows for formatting "pretty
// tables".
package brimtext

import (
	"bytes"
	"strconv"
	"strings"
)

// OrdinalSuffix returns "st", "nd", "rd", etc. for the number given (1st, 2nd,
// 3rd, etc.).
func OrdinalSuffix(number int) string {
	if (number/10)%10 == 1 || number%10 > 3 {
		return "th"
	} else if number%10 == 1 {
		return "st"
	} else if number%10 == 2 {
		return "nd"
	} else if number%10 == 3 {
		return "rd"
	}
	return "th"
}

// ThousandsSep returns the number formatted using the separator at each
// thousands position, such as ThousandsSep(1234567, ",") giving 1,234,567.
func ThousandsSep(v int64, sep string) string {
	s := strconv.FormatInt(v, 10)
	for i := len(s) - 3; i > 0; i -= 3 {
		s = s[:i] + "," + s[i:]
	}
	return s
}

type humanSize struct {
	d int64
	s string
}

var humanSizes = []humanSize{
	humanSize{int64(1024), "K"},
	humanSize{int64(1024) << 10, "M"},
	humanSize{int64(1024) << 20, "G"},
	humanSize{int64(1024) << 30, "T"},
	humanSize{int64(1024) << 40, "P"},
	humanSize{int64(1024) << 50, "E"},
}

// Returns a more readable size format, such as HumanSize(1234567, "") giving
// "1M". For values less than 1K, it is common that no suffix letter should be
// added; but the appendBytes parameter is provided in case clarity is needed.
func HumanSize(b int64, appendBytes string) string {
	if b < 1024 {
		v := strconv.FormatInt(b, 10)
		if appendBytes != "" {
			return v + appendBytes
		}
		return v
	}
	c := b
	s := appendBytes
	for _, h := range humanSizes {
		c = b / h.d
		r := b % h.d
		if r >= h.d/2 {
			c++
		}
		if c < 1024 {
			s = h.s
			break
		}
	}
	return strconv.FormatInt(c, 10) + s
}

type Alignment int

const (
	Left Alignment = iota
	Right
	Center
)

type AlignOptions struct {
	// Widths indicate the desired widths of each column. If nil or if a value
	// is 0, no rewrapping will be done.
	Widths     []int
	Alignments []Alignment
	// FirstDR etc. control what is output for situations with a prepended
	// display row, First row output with Down and Right connections, etc.
	FirstDR       string
	FirstLR       string
	FirstFirstDLR string
	FirstDLR      string
	FirstDL       string
	// RowFirstUD etc. control situations for each data row output.
	RowFirstUD  string
	RowSecondUD string
	RowUD       string
	RowLastUD   string
	// LeaveTrailingWhitespace should be set true if the last cell of data row
	// needs spaces to fill to the end (usually needed when setting RawLastUD).
	LeaveTrailingWhitespace bool
	// FirstNilFirstUDR etc. control situations when the first nil data row is
	// encountered. Can be used to separate the header from the rest of the
	// rows.
	FirstNilFirstUDR  string
	FirstNilLR        string
	FirstNilFirstUDLR string
	FirstNilUDLR      string
	FirstNilLastUDL   string
	// NilFirstUDR etc. control situations when the second and subsequent nil
	// data rows are encountered. Can be used to separate rows from each other.
	NilFirstUDR  string
	NilLR        string
	NilFirstUDLR string
	NilUDLR      string
	NilLastUDL   string
	// LastUR etc. control what is output for situations with an appended
	// display row.
	LastUR       string
	LastLR       string
	LastFirstULR string
	LastULR      string
	LastUL       string
	// NilBetweenEveryRow will add a nil data row between all rows; use to emit
	// FirstNil* and Nil* row separators.
	NilBetweenEveryRow bool
}

// NewDefaultAlignOptions gives:
//
//  &AlignOptions{RowSecondUD: " ", RowUD: " "}
//
// Which will format tables like:
//
//           Bob         Sue    John
//  Hometown San Antonio Austin New York
//  Mother   Bessie      Mary   Sarah
//  Father   Rick        Dan    Mike
func NewDefaultAlignOptions() *AlignOptions {
	return &AlignOptions{RowSecondUD: " ", RowUD: " "}
}

// NewSimpleAlignOptions gives:
//
//  return &AlignOptions{
//      FirstDR:                 "+-",
//      FirstLR:                 "-",
//      FirstFirstDLR:           "-+-",
//      FirstDLR:                "-+-",
//      FirstDL:                 "-+",
//      RowFirstUD:              "| ",
//      RowSecondUD:             " | ",
//      RowUD:                   " | ",
//      RowLastUD:               " |",
//      LeaveTrailingWhitespace: true,
//      FirstNilFirstUDR:        "+-",
//      FirstNilLR:              "-",
//      FirstNilFirstUDLR:       "-+-",
//      FirstNilUDLR:            "-+-",
//      FirstNilLastUDL:         "-+",
//      LastUR:                  "+-",
//      LastLR:                  "-",
//      LastFirstULR:            "-+-",
//      LastULR:                 "-+-",
//      LastUL:                  "-+",
//  }
//
// Which will format tables like:
//
//  +----------+-------------+--------+----------+
//  |          | Bob         | Sue    | John     |
//  +----------+-------------+--------+----------+
//  | Hometown | San Antonio | Austin | New York |
//  | Mother   | Bessie      | Mary   | Sarah    |
//  | Father   | Rick        | Dan    | Mike     |
//  +----------+-------------+--------+----------+
func NewSimpleAlignOptions() *AlignOptions {
	return &AlignOptions{
		FirstDR:                 "+-",
		FirstLR:                 "-",
		FirstFirstDLR:           "-+-",
		FirstDLR:                "-+-",
		FirstDL:                 "-+",
		RowFirstUD:              "| ",
		RowSecondUD:             " | ",
		RowUD:                   " | ",
		RowLastUD:               " |",
		LeaveTrailingWhitespace: true,
		FirstNilFirstUDR:        "+-",
		FirstNilLR:              "-",
		FirstNilFirstUDLR:       "-+-",
		FirstNilUDLR:            "-+-",
		FirstNilLastUDL:         "-+",
		LastUR:                  "+-",
		LastLR:                  "-",
		LastFirstULR:            "-+-",
		LastULR:                 "-+-",
		LastUL:                  "-+",
	}
}

{'fraction_non_alphanumeric': 0.14356887818649838, 'fraction_numerical': 0.01959644606825542, 'mean_word_length': 1.8179691155825923, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5666773', 'n_tokens_mistral': 4091, 'n_tokens_neox': 3742, 'n_words': 1159}