<commit_before>package chunker

import (
    "os"
    "io"
    "strconv"
)

func Chunk(path string) {
    fileinfo,err := os.Stat(path)
    if err != nil {
        panic(err)
    }
    
    if fileinfo.Size() < 100000000 {
        panic(`File must be 100MB or more`)
    }
    
    // open input file
    fi, err := os.Open(path)
    if err != nil { panic(err) }
    // close fi on exit and check for its returned error
    defer func() {
        if err := fi.Close(); err != nil {
            panic(err)
        }
    }()

    n := 1  
    i := 0
    for {
        // open output file
        fo, err := os.Create(path+`.part`+strconv.Itoa(i))
        if err != nil { panic(err) }
        // close fo on exit and check for its returned error
        defer func() {
            if err := fo.Close(); err != nil {
                panic(err)
            }
        }()

        // make a buffer to keep chunks that are read
        buf := make([]byte, 50000000)
        for {
            // read a chunk
            n, err = fi.ReadAt(buf, int64(n-1))
            if err != nil && err != io.EOF { panic(err) }
            if n == 0 { return }

            // write a chunk
            if _, err := fo.Write(buf[:n]); err != nil {
                panic(err)
            }
        }
        
        i++
    }
}

func Reconstruct(path string) {

}<commit_msg>File chunking works, next is reconstructing<commit_after>package chunker

import (
    "os"
    "strconv"
    "log"
)

func Chunk(path string) {
    fileinfo,err := os.Stat(path)
    if err != nil {
        panic(err)
    }
    
    if fileinfo.Size() < 100000000 {
        panic(`File must be 100MB or more`)
    }
    
    // open input file
    fi, err := os.Open(path)
    if err != nil { panic(err) }
    // close fi on exit and check for its returned error
    defer func() {
        if err := fi.Close(); err != nil {
            panic(err)
        }
    }()

    readIndex := 0  
    i := 0
    for {
        // open output file
        fo, err := os.Create(path+`.part`+strconv.Itoa(i))
        if err != nil { panic(err) }
        // close fo on exit and check for its returned error
        defer func() {
            finfo,err := fo.Stat()
            if err != nil { panic(err) } 
        
            if err := fo.Close(); err != nil {
                panic(err)
            }            
                       
            if finfo.Size() == 0 {
                os.Remove(fo.Name())
            }
        }()

        // make a buffer to keep chunks that are read
        buf := make([]byte, 50000000)
        // read a chunk
        log.Println(`readIndex =`, readIndex)
        byteRead, err := fi.ReadAt(buf, int64(readIndex))    
        log.Println(`Read`, byteRead, `bytes`)
        // Somehow ReadAt() returns EOF != io.EOF so we have to manually compare like this
        if err != nil && err.Error() != `read `+fi.Name()+`: Reached the end of the file.` { panic(err) }
        if byteRead == 0 { break }

        // write a chunk
        log.Println(`Begin writing`)
        if _, err := fo.Write(buf[:byteRead]); err != nil {
            panic(err)
        }
        
        readIndex += byteRead        
        i++
    }
}

func Reconstruct(path string) {

}
{'fraction_non_alphanumeric': 0.13174946004319654, 'fraction_numerical': 0.016044430731255787, 'mean_word_length': 1.4014814814814816, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '919060', 'n_tokens_mistral': 1052, 'n_tokens_neox': 986, 'n_words': 299}