<commit_before>package rope

import (
	"bytes"
	"fmt"
	"io"
)

const small = 128

// A rope is a heavy-weight string. They are useful for manipulating very
// long strings.
//
// Like Go strings, ropes are immutable sequences of bytes.
type Rope struct {
	root tree
}

// Make a new rope with contents s.
func New(s string) Rope {
	return Rope{leaf(s)}
}

// Concatenate zero or more ropes.
func Concat(rs ...Rope) Rope {
	switch n := len(rs); n {
	case 0:
		return Rope{leaf("")}
	case 1:
		return rs[0]
	default:
		n2 := n / 2
		r1, r2 := Concat(rs[:n2]...), Concat(rs[n2:]...)
		return Rope{r1.root.concat(r2.root)}
	}
}

// Returns the i'th byte in the rope.
func (r Rope) Index(i int) byte {
	return r.root.index(i)
}

// The length of the rope, in bytes.
func (r Rope) Len() int {
	return r.root.length()
}

// Slice of a rope. Equivalent to New(r.String())[i:j].
func (r Rope) Slice(i, j int) Rope {
	if i == j {
		return Rope{leaf("")}
	}
	return Rope{r.root.slice(i, j)}
}
<commit_msg>faster Concat<commit_after>package rope

import (
	"bytes"
	"fmt"
	"io"
)

const small = 128

// A rope is a heavy-weight string. They are useful for manipulating very
// long strings.
//
// Like Go strings, ropes are immutable sequences of bytes.
type Rope struct {
	root tree
}

// Make a new rope with contents s.
func New(s string) Rope {
	return Rope{leaf(s)}
}

// Concatenate zero or more ropes.
func Concat(ropes ...Rope) Rope {
	switch len(ropes) {
	case 0:
		return Rope{leaf("")}
	default:
		return Rope{concat(ropes)}
	}
}

func concat(ropes []Rope) tree {
	switch n := len(ropes); n {
	case 1:
		return ropes[0].root
	case 2:
		return ropes[0].root.concat(ropes[1].root)
	default:
		n2 := n / 2
		r1, r2 := concat(ropes[:n2]), concat(ropes[n2:])
		return r1.concat(r2)
	}
}

// Returns the i'th byte in the rope.
func (r Rope) Index(i int) byte {
	return r.root.index(i)
}

// The length of the rope, in bytes.
func (r Rope) Len() int {
	return r.root.length()
}

// Slice of a rope. Equivalent to New(r.String())[i:j].
func (r Rope) Slice(i, j int) Rope {
	if i == j {
		return Rope{leaf("")}
	}
	return Rope{r.root.slice(i, j)}
}

{'fraction_non_alphanumeric': 0.1570827489481066, 'fraction_numerical': 0.014492753623188406, 'mean_word_length': 3.1153846153846154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '26703', 'n_tokens_mistral': 902, 'n_tokens_neox': 864, 'n_words': 278}