<commit_before>package main

import (
	"flag"
	"io"
	"bufio"
	"os"
	"encoding/json"
	"net/http"
	"log"
)

func ApiJsonRoundTrip(in io.Reader, out io.Writer, url, username, password, countReq, countGot, countTotal, countScale string) (err error) {
	var request, response interface{}
	var requestString, responseString []byte
	var current, total int
	decoder := json.NewDecoder(in)
	requestString, err := http.NewRequest("POST", url, requestBody)
	if username != "" && password != "" {
		requestString.SetBasicAuth(username, password)
	}

	for decoder.More() {
		err = decoder.Decode(&request)
		if err != nil {
			return err
		}

		current = request["params"]["page_num"].(int)

		for total == 0 || current < total {

			request["params"]["page_num"] = current
			requestString, err = json.Marshal(request)
			if err != nil {
				log.Fatalf("failed to build request body - %v\n%s", err, request)
			}

			client.Body = writeBuf.(io.ReadCloser)
			responseString, err := client.Do(requestString)
			if err != nil {
				log.Fatalf("failed to run request - %v", err)
			}

			err = json.Decode(responseString, &response)
			if err != nil {
				log.Fatalf("failed to decode the response body - %v\n%q", err, responseString)
			}
			current++
			total = reasponse["page_total"]
			out.Write(responseString)
		}
	}
	if err == io.EOF {
		return nil
	} else {
		return err
	}
}

var username = flag.String("username", "", "username to use for authentication")
var password = flag.String("username", "", "username to use for authentication")

func main() {
	url := flag.String("url", "", "url location to direct POSt")
	username := flag.String("username", "", "username to use for authentication")
	password := flag.String("username", "", "username to use for authentication")

	countReq := 
	flag.Parse()

	options := map[string]interface{}{
		"username": username,
		"password": password,
		"url": url,

	}

	if err := PrettyPrint(bufio.NewReader(os.Stdin), bufio.NewWriter(os.Stdout), options); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>wrote function to loop through request<commit_after>package main

import (
	"flag"
	"io"
	"bufio"
	"bytes"
	"os"
	"encoding/json"
	"golang.org/x/net/publicsuffix"
	"net/http"
	"log"
)

func loopRequest(requestData interface{}, out io.Writer, username, password, url string, locReq, locCur, locTotal []string, incPage int) (error) {

  jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
  if err != nil {
      log.Fatal(err)
  }
  client := http.Client{Jar: jar}

  requestBytes, err := json.Marshal(requestData)
  if err != nil {
  	return err
  }

  request, err := http.NewRequest("POST", url, bytes.NewReader(requestBytes))
  if err != nil {
  	return err
  }

  if username != "" && password != "" {
  	request.SetBasicAuth(username, password)
  }

  responseBytes, err := client.Do(request)
  if err != nil {
  	return err
  }

  var responseData interface{}

  err = json.Unmarshal(responseBytes, responseData)
  if err != nil {
  	return err
  }

  var reqPage, curPage, totalPage int

  reqPage, err = cliTricks.GetInt(requestData, locPage)
  if err != nil {
  	return fmt.Errorf("bad request page - %v", err)
  }

  curPage, err = cliTricks.GetInt(requestData, locCur)
  if err != nil {
  	return fmt.Errorf("bad current page - %v", err)
  }

  totalPage, err = cliTricks.GetInt(requestData, locTotal)
  if err != nil {
  	return fmt.Errorf("bad current page - %v", err)
	}

  for curPage < totalPage {
  	curPage += incPage
  	err = cliTricks.SetItem(requestData, curPage, locCur)
  	if err != nil {
  		fmt.ErrorF("failed to set the current page - %v", err)
  	}

	  requestBytes, err = json.Marshal(requestData)
	  if err != nil {
	  	return err
	  }

	  request.Body = bytes.NewReader(requestBytes)
	  responseBytes, err := client.Do(request)
	  if err != nil {
	  	return err
	  }

	  err = json.Unmarshal(responseBytes, responseData)
	  if err != nil {
	  	return err
	  }

	  curPage, err = cliTricks.GetInt(requestData, locCur)
	  if err != nil {
	  	return fmt.Errorf("bad current page - %v", err)
	  }
  }
  return nil
}

func ApiJsonRoundTrip(in io.Reader, out io.Writer, url, username, password string, countReq, countGot, countTotal []string) (err error) {
	var request, response interface{}
	var requestBytes, responseBytes []byte
	var current, total int

  jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
  if err != nil {
      log.Fatal(err)
  }
  client := http.Client{Jar: jar}

	decoder := json.NewDecoder(in)

	requestClient, err := http.NewRequest("POST", url, bytes.NewReader(requestBytes))
	if username != "" && password != "" {
		requestClient.SetBasicAuth(username, password)
	}

	for decoder.More() {
		err = decoder.Decode(&request)
		if err != nil {
			return err
		}

		current, err = cliTricks.getItem(request, []string{"params","page_num"})
		if err != nil {
			log.Fatalf("failed to retrieve item - %v", err)
		}

		if current, ok := current.(int); !ok {

		}

		for total == 0 || current < total {

			request["params"]["page_num"] = current
			requestBytes, err = json.Marshal(request)
			if err != nil {
				log.Fatalf("failed to build request body - %v\n%s", err, request)
			}

			client.Body = bytes.NewReader(requestBytes)
			responseBytes, err := client.Do(requestBytes)
			if err != nil {
				log.Fatalf("failed to run request - %v", err)
			}

			err = json.Decode(responseBytes, &response)
			if err != nil {
				log.Fatalf("failed to decode the response body - %v\n%q", err, responseBytes)
			}
			current++
			total = reasponse["page_total"]
			out.Write(responseBytes)
		}
	}
	if err == io.EOF {
		return nil
	} else {
		return err
	}
}

func main() {
	url := flag.String("url", "", "url location to direct POSt")
	username := flag.String("username", "", "username to use for authentication")
	password := flag.String("username", "", "username to use for authentication")

	countReq := 
	flag.Parse()

	options := map[string]interface{}{
		"username": username,
		"password": password,
		"url": url,

	}

	if err := PrettyPrint(bufio.NewReader(os.Stdin), bufio.NewWriter(os.Stdout), options); err != nil {
		log.Fatal(err)
	}
}

{'fraction_non_alphanumeric': 0.14562792952965897, 'fraction_numerical': 0.0003232584451268789, 'mean_word_length': 3.6007434944237917, 'pattern_counts': {'":': 6, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1411176', 'n_tokens_mistral': 2242, 'n_tokens_neox': 2060, 'n_words': 550}