<commit_before>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"math/big"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gosuri/uilive"

	"math/rand" // for debug purposes
)

var debugging = false // if true, debug messages will be shown

var (
	res          Resumer
	outputWriter *bufio.Writer

	resumerSuffix string = ".audisto_"
)

// flags
var (
	username string
	password string
	crawl    uint64

	noDetails bool
	output    string
	noResume  bool
)

// progress bar elements
var (
	progressIndicator *uilive.Writer
	progressStatus    string

	timeoutCount int
	errorCount   int

	averageTimePer1000 float64 = 1
)

type Resumer struct {
	OutputFilename string `json:"outputFilename"`

	chunkSize     int64
	DoneElements  int64 `json:"doneElements"`
	TotalElements int64 `json:"totalElements"`
	NoDetails     bool  `json:"noDetails"`

	httpClient http.Client
}

// chunk is used to get unmarshal the json containing the total number of chunks
type chunk struct {
	Chunk struct {
		Total int64 `json:"total"`
		Page  int   `json:"page"`
		Size  int   `json:"size"`
	} `json:"chunk"`
}

// init parses flags and sets everything up
func init() {

	flag.StringVar(&username, "username", "", "API Username (required)")
	flag.StringVar(&password, "password", "", "API Password (required)")
	flag.Uint64Var(&crawl, "crawl", 0, "ID of the crawl to download (required)")

	flag.BoolVar(&noDetails, "no-details", false, "If passed, details in API request is set to 0 else")
	flag.StringVar(&output, "output", "", "Path for the output file")
	flag.BoolVar(&noResume, "no-resume", false, "If passed, download starts again, else the download is resumed")

	flag.Usage = usage
	flag.Parse()

	username = strings.TrimSpace(username)
	password = strings.TrimSpace(password)
	output = strings.TrimSpace(output)

	// Check for non-valid flags
	usernameIsNull := username == ""
	passwordIsNull := password == ""
	crawlIsNull := crawl == 0

	if usernameIsNull || passwordIsNull || crawlIsNull {
		usage()
		os.Exit(0)
	}

	// stdout or output file ?
	if output == "" {
		outputWriter = bufio.NewWriter(os.Stdout)

		var err error

		res = Resumer{}

		res.TotalElements, err = totalElements()
		if err != nil {
			fmt.Println(err)
			return
		}
		res.OutputFilename = output
		res.NoDetails = noDetails
	} else {

		errOutput, errResumer := fExists(output), fExists(output+resumerSuffix)
		startAnew := errOutput != nil && errResumer != nil

		// if don't resume, create new set
		if noResume || startAnew {

			if startAnew && !noResume {
				fmt.Println("No download to resume; stating new.")
			}

			// if resume, check if output file exists
			if errOutput == nil {
				fmt.Println("File already exists; please resume removing --no-resume, delete or specify another output filename.")
				return
			}

			var err error

			res = Resumer{}

			res.TotalElements, err = totalElements()
			if err != nil {
				fmt.Println(err)
				return
			}
			res.OutputFilename = output
			res.NoDetails = noDetails

			err = res.PersistConfig()
			if err != nil {
				panic(err)
			}

			// create new outputFile
			newFile, err := os.Create(output)
			if err != nil {
				panic(err)
			}
			outputWriter = bufio.NewWriter(newFile)
		} else {
			// if resume, check if output file exists
			if errOutput != nil {
				fmt.Println(fmt.Sprintf("Cannot resume; %q file does not exist: use --no-resume to create new.", output))
				return
			}
			// if resume, check if resume file exists
			if errResumer != nil {
				fmt.Println(fmt.Sprintf("Cannot resume; resumer file %v does not exist: ", output+resumerSuffix))
				return
			}

			resumerFile, err := ioutil.ReadFile(output + resumerSuffix)
			if err != nil {
				panic(fmt.Sprintf("Resumer file error: %v\n", err))
			}
			err = json.Unmarshal(resumerFile, &res)
			if err != nil {
				panic(fmt.Sprintf("Resumer file error: %v\n", err))
			}

			// open outputFile
			existingFile, err := os.OpenFile(output, os.O_WRONLY|os.O_APPEND, 0777)
			if err != nil {
				panic(err)
			}
			outputWriter = bufio.NewWriter(existingFile)

			// read and validate resumer file
			// read and validate output file
			// check last id of the last write batch

			if res.NoDetails != noDetails {
				fmt.Println(fmt.Sprintf("Warning! This file was begun with --no-details=%v; continuing with --no-details=%v will break the file.", res.NoDetails, noDetails))
				return
			}

		}

	}

	// set chunkSize to 10000
	res.chunkSize = 10000
}

func main() {

	// only show progress bar when downloading to file
	if output != "" {
		progressIndicator = uilive.New()
		progressIndicator.Start()
		go progressLoop()
	}

	debug(username, password, crawl)
	debugf("%#v\n", res)

MainLoop:<commit_msg>on error inside init() -> os.Exit(0)<commit_after>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"math/big"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gosuri/uilive"

	"math/rand" // for debug purposes
)

var debugging = false // if true, debug messages will be shown

var (
	res          Resumer
	outputWriter *bufio.Writer

	resumerSuffix string = ".audisto_"
)

// flags
var (
	username string
	password string
	crawl    uint64

	noDetails bool
	output    string
	noResume  bool
)

// progress bar elements
var (
	progressIndicator *uilive.Writer
	progressStatus    string

	timeoutCount int
	errorCount   int

	averageTimePer1000 float64 = 1
)

type Resumer struct {
	OutputFilename string `json:"outputFilename"`

	chunkSize     int64
	DoneElements  int64 `json:"doneElements"`
	TotalElements int64 `json:"totalElements"`
	NoDetails     bool  `json:"noDetails"`

	httpClient http.Client
}

// chunk is used to get unmarshal the json containing the total number of chunks
type chunk struct {
	Chunk struct {
		Total int64 `json:"total"`
		Page  int   `json:"page"`
		Size  int   `json:"size"`
	} `json:"chunk"`
}

// init parses flags and sets everything up
func init() {

	flag.StringVar(&username, "username", "", "API Username (required)")
	flag.StringVar(&password, "password", "", "API Password (required)")
	flag.Uint64Var(&crawl, "crawl", 0, "ID of the crawl to download (required)")

	flag.BoolVar(&noDetails, "no-details", false, "If passed, details in API request is set to 0 else")
	flag.StringVar(&output, "output", "", "Path for the output file")
	flag.BoolVar(&noResume, "no-resume", false, "If passed, download starts again, else the download is resumed")

	flag.Usage = usage
	flag.Parse()

	username = strings.TrimSpace(username)
	password = strings.TrimSpace(password)
	output = strings.TrimSpace(output)

	// Check for non-valid flags
	usernameIsNull := username == ""
	passwordIsNull := password == ""
	crawlIsNull := crawl == 0

	if usernameIsNull || passwordIsNull || crawlIsNull {
		usage()
		os.Exit(0)
	}

	// stdout or output file ?
	if output == "" {
		outputWriter = bufio.NewWriter(os.Stdout)

		var err error

		res = Resumer{}

		res.TotalElements, err = totalElements()
		if err != nil {
			fmt.Println(err)
			os.Exit(0)
		}
		res.OutputFilename = output
		res.NoDetails = noDetails
	} else {

		errOutput, errResumer := fExists(output), fExists(output+resumerSuffix)
		startAnew := errOutput != nil && errResumer != nil

		// if don't resume, create new set
		if noResume || startAnew {

			if startAnew && !noResume {
				fmt.Println("No download to resume; stating new.")
			}

			// if resume, check if output file exists
			if errOutput == nil {
				fmt.Println("File already exists; please resume removing --no-resume, delete or specify another output filename.")
				os.Exit(0)
			}

			var err error

			res = Resumer{}

			res.TotalElements, err = totalElements()
			if err != nil {
				fmt.Println(err)
				os.Exit(0)
			}
			res.OutputFilename = output
			res.NoDetails = noDetails

			err = res.PersistConfig()
			if err != nil {
				panic(err)
			}

			// create new outputFile
			newFile, err := os.Create(output)
			if err != nil {
				panic(err)
			}
			outputWriter = bufio.NewWriter(newFile)
		} else {
			// if resume, check if output file exists
			if errOutput != nil {
				fmt.Println(fmt.Sprintf("Cannot resume; %q file does not exist: use --no-resume to create new.", output))
				os.Exit(0)
			}
			// if resume, check if resume file exists
			if errResumer != nil {
				fmt.Println(fmt.Sprintf("Cannot resume; resumer file %v does not exist: ", output+resumerSuffix))
				os.Exit(0)
			}

			resumerFile, err := ioutil.ReadFile(output + resumerSuffix)
			if err != nil {
				panic(fmt.Sprintf("Resumer file error: %v\n", err))
			}
			err = json.Unmarshal(resumerFile, &res)
			if err != nil {
				panic(fmt.Sprintf("Resumer file error: %v\n", err))
			}

			// open outputFile
			existingFile, err := os.OpenFile(output, os.O_WRONLY|os.O_APPEND, 0777)
			if err != nil {
				panic(err)
			}
			outputWriter = bufio.NewWriter(existingFile)

			// read and validate resumer file
			// read and validate output file
			// check last id of the last write batch

			if res.NoDetails != noDetails {
				fmt.Println(fmt.Sprintf("Warning! This file was begun with --no-details=%v; continuing with --no-details=%v will break the file.", res.NoDetails, noDetails))
				os.Exit(0)
			}

		}

	}

	// set chunkSize to 10000
	res.chunkSize = 10000
}

func main() {

	// only show progress bar when downloading to file
	if output != "" {
		progressIndicator = uilive.New()
		progressIndicator.Start()
		go progressLoop()
	}

	debug(username, password, crawl)
	debugf("%#v\n", res)

MainLoop:
{'fraction_non_alphanumeric': 0.12317111134170385, 'fraction_numerical': 0.008405105323233372, 'mean_word_length': 3.3026785714285714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '148129', 'n_tokens_mistral': 3624, 'n_tokens_neox': 3164, 'n_words': 939}