<commit_before>package huffman

import (
    "io"
    "fmt"
    "bufio"
    "bytes"
)

const (
    BUFFER_SIZE int = 65536
    LEAF int = 1024
)


type huffmanNode struct {
    index int
    parent *huffmanNode
    left *huffmanNode
    right *huffmanNode
}

type Huffman struct {
    length uint64
    encoding *map[int][]byte
    root *huffmanNode
    leaves []*huffmanNode
}

func New(length uint64) *Huffman {
    leaves := make([]*huffmanNode, 256)
    return &Huffman{length: length, leaves: leaves}
}

func (self *Huffman) Encode(reader io.Reader, writer io.Writer) error {
    if err := self.buildEncodingTree(reader); err != nil {
        return err
    }

    if err := self.buildEncodingMap(); err != nil {
        return err
    }

    buffer := make([]byte, 1024)
    actualLen := self.serializeTree(self.root, buffer)
    fmt.Printf("SERIALIZED TREE: %v", buffer[:actualLen])
    writer.Write(buffer[:actualLen])

    return nil
}

func (self *Huffman) Decode(reader io.Reader, writer io.Writer) error {
    return nil
}

func (self *Huffman) buildEncodingTree(reader io.Reader) error {
    rates, total, start := make([]uint16, 512), uint64(0), 0
    bReader, buf := bufio.NewReader(reader), make([]byte, BUFFER_SIZE)

    for total < self.length {
        if obtained, err := bReader.Read(buf); err == nil {
            total += uint64(obtained)
            for _, symbol := range buf[:obtained]{<commit_msg>enconding<commit_after>package huffman

import (
    "io"
    "os"
    "fmt"
    "bytes"
    "errors"
)

const (
    BUFFER_SIZE uint = 65536
    LEAF int = 1024
)


type huffmanNode struct {
    index int
    parent *huffmanNode
    left *huffmanNode
    right *huffmanNode
}

type bytesBuffer struct {
    byteBuf []byte
    bitBuf byte
    byteBufPos uint
    bitBufPos uint
}

type Huffman struct {
    length uint64
    encoding *map[int][]byte
    root *huffmanNode
    leaves []*huffmanNode
}

func New(length uint64) *Huffman {
    leaves := make([]*huffmanNode, 256)
    return &Huffman{length: length,leaves: leaves}
}

func (self *Huffman) Encode(reader io.ReadSeeker, writer io.Writer) error {
    if err := self.buildEncodingTree(reader); err != nil {
        return err
    }

    if err := self.buildEncodingMap(); err != nil {
        return err
    }

    buffer := make([]byte, 2048)
    actualLen := self.serializeTree(self.root, buffer)
    fmt.Printf("SERIALIZED TREE: %v", buffer[:actualLen])
    writer.Write(buffer[:actualLen])

    reader.Seek(0, os.SEEK_SET)
    raw, total := make([]byte, BUFFER_SIZE), uint64(0)
    buf := &bytesBuffer{byteBuf: make([]byte, BUFFER_SIZE),
                        bitBufPos: 7}

    for total < self.length {
        if obtained, err := reader.Read(raw); err == nil {
            total += uint64(obtained)
            if self.encodePartial(writer, raw[:obtained], buf) != nil {
                return err
            }
        } else if err == io.EOF {
            break
        } else {
            return err
        }
    }

    return nil
}

func (self *Huffman) Decode(reader io.Reader, writer io.Writer) error {
    return nil
}

func (self *Huffman) encodePartial(writer io.Writer, raw []byte, buf *bytesBuffer) error {
    if self.encoding == nil {
        return errors.New("Unexpected error. Invalid encoding table.")
    }

    for _, symbol := range raw {
        encoding := (*self.encoding)[int(symbol)]

        for index := range encoding {
            bit := encoding[len(encoding)-index-1]
            buf.bitBuf |= (bit & 1) << buf.bitBufPos

            if buf.bitBufPos--; buf.bitBufPos == 0 {
                buf.byteBuf[buf.byteBufPos] = buf.bitBuf

                if buf.byteBufPos++; buf.byteBufPos == BUFFER_SIZE {
                    if _, err := writer.Write(buf.byteBuf); err != nil {
                        return err
                    }

                    buf.byteBufPos = 0
                }

                buf.bitBuf = 0
                buf.bitBufPos = 7
            }
        }
    }

    return nil
}

func (self *Huffman) decodePartial() error {
    return nil
}

func (self *Huffman) buildEncodingTree(reader io.Reader) error {
    rates, total, start := make([]uint16, 512), uint64(0), 0
    buf := make([]byte, BUFFER_SIZE)

    for total < self.length {
        if obtained, err := reader.Read(buf); err == nil {
            total += uint64(obtained)
            for _, symbol := range buf[:obtained]{
{'fraction_non_alphanumeric': 0.12091134671779834, 'fraction_numerical': 0.01691856530566208, 'mean_word_length': 2.2626931567328916, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4985715', 'n_tokens_mistral': 1482, 'n_tokens_neox': 1391, 'n_words': 366}