<commit_before>package lshensemble

import (
	"encoding/binary"
	"io"
	"sort"
)

type Set struct {
	Data map[string]bool
}

func NewSet() *Set {
	return &Set{
		Data: make(map[string]bool),
	}
}

// Add new element to the set
func (s *Set) Add(t string) {
	s.Data[t] = true
}

// len()
func (s *Set) Len() int {
	return len(s.Data)
}

// DomainRecord represents a domain record.
type DomainRecord struct {
	// The unique key of this domain.
	Key string
	// The domain size.
	Size int
	// The MinHash signature of this domain.
	Signature Signature
}

// Write serializes the domain record and write to the data stream w, which implements
// the io.Writer interface.
// keyFn is used to convert the string key into bytes.
// Write returns the number of bytes wrote and error if encountered any.
func (r *DomainRecord) Write(w io.Writer, keyFn func(string) ([]byte, error)) (int, error) {
	keyBin, err := keyFn(r.Key)
	if err != nil {
		return 0, err
	}
	n, err := w.Write(keyBin)
	if err != nil {
		return n, err
	}
	size := int64(r.Size)
	if err := binary.Write(w, binary.LittleEndian, size); err != nil {
		return n, err
	}
	n += 8
	length := int32(len(r.Signature))
	if err := binary.Write(w, binary.LittleEndian, length); err != nil {
		return n, err
	}
	n += 4
	for i := range r.Signature {
		if err := binary.Write(w, binary.LittleEndian, r.Signature[i]); err != nil {
			return n, err
		}
		n += 8
	}
	return n, nil
}

// Read deserializes the domain record from a data stream reader.
// keySize is the number of bytes of the serialized key.
// keyFn is used to convert the serialized key into string.
// Read returns the number of bytes read and error if encountered any.
func (r *DomainRecord) Read(reader io.Reader, keySize int, keyFn func([]byte) (string, error)) (int, error) {
	keyBin := make([]byte, keySize)
	n, err := reader.Read(keyBin)
	if err != nil {
		return n, err
	}
	key, err := keyFn(keyBin)
	if err != nil {
		return n, err
	}
	r.Key = key
	var size int64
	if err := binary.Read(reader, binary.LittleEndian, &size); err != nil {
		return n, err
	}
	r.Size = int(size)
	n += 8
	var length int32
	if err := binary.Read(reader, binary.LittleEndian, &length); err != nil {
		return n, err
	}
	n += 4
	r.Signature = make(Signature, int(length))
	for i := range r.Signature {
		if err := binary.Read(reader, binary.LittleEndian, &(r.Signature[i])); err != nil {
			return n, err
		}
		n += 8<commit_msg>use big endian allow external sort<commit_after>package lshensemble

import (
	"encoding/binary"
	"io"
	"sort"
)

type Set struct {
	Data map[string]bool
}

func NewSet() *Set {
	return &Set{
		Data: make(map[string]bool),
	}
}

// Add new element to the set
func (s *Set) Add(t string) {
	s.Data[t] = true
}

// len()
func (s *Set) Len() int {
	return len(s.Data)
}

// DomainRecord represents a domain record.
type DomainRecord struct {
	// The unique key of this domain.
	Key string
	// The domain size.
	Size int
	// The MinHash signature of this domain.
	Signature Signature
}

// Write serializes the domain record and write to the data stream w, which implements
// the io.Writer interface.
// keyFn is used to convert the string key into bytes.
// Write returns the number of bytes wrote and error if encountered any.
// All integers are serialized according to Big Endian.
func (r *DomainRecord) Write(w io.Writer, keyFn func(string) ([]byte, error)) (int, error) {
	keyBin, err := keyFn(r.Key)
	if err != nil {
		return 0, err
	}
	n, err := w.Write(keyBin)
	if err != nil {
		return n, err
	}
	size := int64(r.Size)
	if err := binary.Write(w, binary.BigEndian, size); err != nil {
		return n, err
	}
	n += 8
	length := int32(len(r.Signature))
	if err := binary.Write(w, binary.BigEndian, length); err != nil {
		return n, err
	}
	n += 4
	for i := range r.Signature {
		if err := binary.Write(w, binary.BigEndian, r.Signature[i]); err != nil {
			return n, err
		}
		n += 8
	}
	return n, nil
}

// Read deserializes the domain record from a data stream reader.
// keySize is the number of bytes of the serialized key.
// keyFn is used to convert the serialized key into string.
// Read returns the number of bytes read and error if encountered any.
// All integers are assumed to be serialized using Big Endian.
func (r *DomainRecord) Read(reader io.Reader, keySize int, keyFn func([]byte) (string, error)) (int, error) {
	keyBin := make([]byte, keySize)
	n, err := reader.Read(keyBin)
	if err != nil {
		return n, err
	}
	key, err := keyFn(keyBin)
	if err != nil {
		return n, err
	}
	r.Key = key
	var size int64
	if err := binary.Read(reader, binary.BigEndian, &size); err != nil {
		return n, err
	}
	r.Size = int(size)
	n += 8
	var length int32
	if err := binary.Read(reader, binary.BigEndian, &length); err != nil {
		return n, err
	}
	n += 4
	r.Signature = make(Signature, int(length))
	for i := range r.Signature {
		if err := binary.Read(reader, binary.BigEndian, &(r.Signature[i])); err != nil {
			return n, err
		}
		n += 8
{'fraction_non_alphanumeric': 0.13255439161966157, 'fraction_numerical': 0.0060435132957292505, 'mean_word_length': 3.2949826989619377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4567380', 'n_tokens_mistral': 1799, 'n_tokens_neox': 1724, 'n_words': 646}