<commit_before>/*
	Copyright 2019 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package filepath

import (
	"fmt"
	"net/http"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

/*
FilePath turns a single date path into the paths of all
	relevant files for that date
*/
type FilePath struct {
	FilePath chan string
	basePath string
	wg       sync.WaitGroup
}

/*
ChannelListener should be run in a goroutine and will receive paths on the FilePath
	channel.  It will retrieve the data for that path and save it to a file in the
	location specified.  Once the channel is closed, the goroutine exits
*/
func (fP *FilePath) ChannelListener(client *http.Client) {
	for inputPath := range fP.FilePath {
		fmt.Printf("\tRequesting %s\n", inputPath)
		fP.wg.Add(1)
		time.Sleep(2 * time.Second)
		resp, err := client.Get(inputPath)
		if err != nil {
			fmt.Println(err.Error())
		} else if i := strings.Index(inputPath, "gid_"); i >= 0 {
			outputPath := filepath.Join(fP.basePath, strings.Replace(inputPath[i:], "/", "_", -1))
			fP.writeFile(outputPath, resp)
		}
		fP.wg.Done()
	}
}

func (fP *FilePath) writeFile(filePath string, resp *http.Response) {
	f, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm)
	if err != nil {
		fmt.Println(err)<commit_msg>Increase time between requests<commit_after>/*
	Copyright 2019 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package filepath

import (
	"fmt"
	"net/http"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

/*
FilePath turns a single date path into the paths of all
	relevant files for that date
*/
type FilePath struct {
	FilePath chan string
	basePath string
	wg       sync.WaitGroup
}

/*
ChannelListener should be run in a goroutine and will receive paths on the FilePath
	channel.  It will retrieve the data for that path and save it to a file in the
	location specified.  Once the channel is closed, the goroutine exits
*/
func (fP *FilePath) ChannelListener(client *http.Client) {
	for inputPath := range fP.FilePath {
		fmt.Printf("\tRequesting %s\n", inputPath)
		fP.wg.Add(1)
		time.Sleep(4 * time.Second)
		resp, err := client.Get(inputPath)
		if err != nil {
			fmt.Println(err.Error())
		} else if i := strings.Index(inputPath, "gid_"); i >= 0 {
			outputPath := filepath.Join(fP.basePath, strings.Replace(inputPath[i:], "/", "_", -1))
			fP.writeFile(outputPath, resp)
		}
		fP.wg.Done()
	}
}

func (fP *FilePath) writeFile(filePath string, resp *http.Response) {
	f, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm)
	if err != nil {
		fmt.Println(err)
{'fraction_non_alphanumeric': 0.10360980986497657, 'fraction_numerical': 0.006613392119041058, 'mean_word_length': 3.6778350515463916, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2609873', 'n_tokens_mistral': 1222, 'n_tokens_neox': 1163, 'n_words': 426}