<commit_before>package main

import (
	"bufio"
	"compress/bzip2"
	"compress/gzip"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"time"
)

//const READ_BUF_SIZE = 10485760 // Set 10MB read buffer
var cpuprofile *string = flag.String("cpuprofile", "", "Write cpu profile to file")
var compfile *string = flag.String("file", "", "Compressed file to decompress")

func main() {
	flag.Parse()

	// Check arguments for sanity and requirements
	switch {
	case *compfile == "":
		log.Fatal("You must provide a compressed file!\n")
	case *cpuprofile != "":
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	log.Printf("Decompressing %s\n", *compfile)
	err := fileOpen(*compfile)
	if err != nil {
		log.Fatalf(" * ERROR: %v\n", err)
	}
	log.Print("Done\n")
}

func fileOpen(path string) error {
	// Open the file and defer its close
	fh, err := os.OpenFile(path, 0, 0)
	if err != nil {
		return err
	}
	defer fh.Close()

	br := bufio.NewReader(fh)

	switch filepath.Ext(path) {
	case ".bz2":
		// Read bzip2 compressed log file
		cr := bzip2.NewReader(br)
		//err := logRead(cr)
		err := dataParse(cr)
		if err != nil {
			return err
		}
	case ".gz":
		// Read gzip compressed log file
		cr, _ := gzip.NewReader(br)
		defer cr.Close()
		err := dataParse(cr)
		if err != nil {
			return err
		}
	default:
		// Read uncompressed log file
		err := dataParse(br)
		if err != nil {
			return err
		}
	}
	return nil
}

func dataParse(r io.Reader) error {
	br := bufio.NewReader(r)

	st := time.Now()
	defer log.Printf(" * Decompress time: %0.2f sec", time.Since(st).Seconds())
	_, e := io.Copy(ioutil.Discard, br)
	if e != nil {
		return e
	}

	return nil
}<commit_msg>Do not defer printing elapsed time<commit_after>package main

import (
	"bufio"
	"compress/bzip2"
	"compress/gzip"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"time"
)

//const READ_BUF_SIZE = 10485760 // Set 10MB read buffer
var cpuprofile *string = flag.String("cpuprofile", "", "Write cpu profile to file")
var compfile *string = flag.String("file", "", "Compressed file to decompress")

func main() {
	flag.Parse()

	// Check arguments for sanity and requirements
	switch {
	case *compfile == "":
		log.Fatal("You must provide a compressed file!\n")
	case *cpuprofile != "":
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	log.Printf("Decompressing %s\n", *compfile)
	err := fileOpen(*compfile)
	if err != nil {
		log.Fatalf(" * ERROR: %v\n", err)
	}
	log.Print("Done\n")
}

func fileOpen(path string) error {
	// Open the file and defer its close
	fh, err := os.OpenFile(path, 0, 0)
	if err != nil {
		return err
	}
	defer fh.Close()

	br := bufio.NewReader(fh)

	switch filepath.Ext(path) {
	case ".bz2":
		// Read bzip2 compressed log file
		cr := bzip2.NewReader(br)
		//err := logRead(cr)
		err := dataParse(cr)
		if err != nil {
			return err
		}
	case ".gz":
		// Read gzip compressed log file
		cr, _ := gzip.NewReader(br)
		defer cr.Close()
		err := dataParse(cr)
		if err != nil {
			return err
		}
	default:
		// Read uncompressed log file
		err := dataParse(br)
		if err != nil {
			return err
		}
	}
	return nil
}

func dataParse(r io.Reader) error {
	br := bufio.NewReader(r)

	st := time.Now()
	_, e := io.Copy(ioutil.Discard, br)
	if e != nil {
		return e
	}
	log.Printf(" * Decompress time: %0.3f sec", time.Since(st).Seconds())
	return nil
}
{'fraction_non_alphanumeric': 0.15756893640967923, 'fraction_numerical': 0.010129431626336522, 'mean_word_length': 3.262589928057554, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5481910', 'n_tokens_mistral': 1502, 'n_tokens_neox': 1385, 'n_words': 370}