<commit_before>package vfsutil

import (
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"

	"golang.org/x/net/context/ctxhttp"

	"github.com/fhs/go-netrc/netrc"
	"github.com/pkg/errors"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
)

// NewZipVFS downloads a zip archive from a URL (or fetches from the local cache
// on disk) and returns a new VFS backed by that zip archive.
func NewZipVFS(ctx context.Context, url string, onFetchStart, onFetchFailed func(), evictOnClose bool) (*ArchiveFS, error) {
	request, err := http.NewRequest("HEAD", url, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to construct a new request with URL %s", url)
	}
	setAuthFromNetrc(request)
	response, err := ctxhttp.Do(ctx, nil, request)
	if err != nil {
		return nil, err
	}
	if response.StatusCode != http.StatusOK {
		return nil, nil
	}

	fetch := func(ctx context.Context) (ar *archiveReader, err error) {<commit_msg>remember to return an error when the HEAD response is not 200 OK<commit_after>package vfsutil

import (
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"

	"golang.org/x/net/context/ctxhttp"

	"github.com/fhs/go-netrc/netrc"
	"github.com/pkg/errors"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
)

// NewZipVFS downloads a zip archive from a URL (or fetches from the local cache
// on disk) and returns a new VFS backed by that zip archive.
func NewZipVFS(ctx context.Context, url string, onFetchStart, onFetchFailed func(), evictOnClose bool) (*ArchiveFS, error) {
	request, err := http.NewRequest("HEAD", url, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to construct a new request with URL %s", url)
	}
	setAuthFromNetrc(request)
	response, err := ctxhttp.Do(ctx, nil, request)
	if err != nil {
		return nil, err
	}
	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("unable to fetch zip from %s (expected HTTP response code 200, but got %d)", url, response.StatusCode)
	}

	fetch := func(ctx context.Context) (ar *archiveReader, err error) {
{'fraction_non_alphanumeric': 0.12610516519311307, 'fraction_numerical': 0.004653327128897162, 'mean_word_length': 4.321782178217822, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5061898', 'n_tokens_mistral': 790, 'n_tokens_neox': 766, 'n_words': 227}