<commit_before>package fetch

import (
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"sync"
	"time"

	"bufio"
	"encoding/binary"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/golang/protobuf/proto"
	"github.com/srom/chessbot/common"
)

const BATCH_SIZE = 1e5
const BUCKET_NAME = "chessbot"
const KEY_FORMAT = "<KEY>"

func FetchData(awsSession *session.Session, done <-chan struct{}, featureChannels ...<-chan *common.ChessBotTriplet) {
	start := time.Now()
	loopStart := time.Now()
	iteration := 0
	var wgOutChan sync.WaitGroup<commit_msg>Fetch inputs to another location<commit_after>package fetch

import (
	"bufio"
	"compress/gzip"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/golang/protobuf/proto"
	"github.com/srom/chessbot/common"
)

const BATCH_SIZE = 1e5
const BUCKET_NAME = "chessbot"
const KEY_FORMAT = "<KEY>"

func FetchData(awsSession *session.Session, done <-chan struct{}, featureChannels ...<-chan *common.ChessBotTriplet) {
	start := time.Now()
	loopStart := time.Now()
	iteration := 0
	var wgOutChan sync.WaitGroup
{'fraction_non_alphanumeric': 0.16113744075829384, 'fraction_numerical': 0.007898894154818325, 'mean_word_length': 4.461206896551724, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3481073', 'n_tokens_mistral': 526, 'n_tokens_neox': 510, 'n_words': 89}