<commit_before>package itchfs

import (
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/go-errors/errors"
	itchio "github.com/itchio/go-itchio"
	"github.com/itchio/httpfile"
)

// over-engineering follows

type SourceType int

const (
	// 691 is the earliest Bernoulli number that's also an irregular prime.
	// It's also sufficiently different from zero that it should surface
	// coding errors more easily
	SourceType_DownloadBuild SourceType = 691 + iota
	SourceType_KeyDownloadBuild
	SourceType_WharfDownloadBuild
)

type Source struct {
	Type       SourceType
	ItchClient *itchio.Client
	Path       string
}

var patterns = map[string]SourceType{
	"/upload/*/download/builds/*":         SourceType_DownloadBuild,
	"/download-key/*/download/*/builds/*": SourceType_KeyDownloadBuild,
	"/wharf/builds/*/files/*/download":    SourceType_WharfDownloadBuild,
}

func ObtainSource(itchClient *itchio.Client, itchPath string) (*Source, error) {
	var matches bool
	var err error

	for pattern, sourceType := range patterns {
		matches, err = path.Match(pattern, itchPath)
		if err != nil {
			return nil, err
		}

		if matches {
			return &Source{
				Type:       sourceType,
				ItchClient: itchClient,
				Path:       itchPath,
			}, nil
		}
	}

	return nil, fmt.Errorf("unrecognized itchfs pattern: %s", itchPath)
}

func (s *Source) makeGetURL() (httpfile.GetURLFunc, error) {
	tokens := strings.Split(s.Path, "/")

	var getter httpfile.GetURLFunc

	switch s.Type {
	case SourceType_DownloadBuild:
		uploadID, err := strconv.ParseInt(tokens[2], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildID, err := strconv.ParseInt(tokens[5], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.DownloadUploadBuild(uploadID, buildID)
			if err != nil {
				return "", err
			}

			return r.Archive.URL, nil
		}
	}

	return getter, nil
}<commit_msg>Add two other strategies for itchfs<commit_after>package itchfs

import (
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/go-errors/errors"
	itchio "github.com/itchio/go-itchio"
	"github.com/itchio/httpfile"
)

// over-engineering follows

type SourceType int

const (
	// 691 is the earliest Bernoulli number that's also an irregular prime.
	// It's also sufficiently different from zero that it should surface
	// coding errors more easily
	SourceType_DownloadBuild SourceType = 691 + iota
	SourceType_KeyDownloadBuild
	SourceType_WharfDownloadBuild
)

type Source struct {
	Type       SourceType
	ItchClient *itchio.Client
	Path       string
}

var patterns = map[string]SourceType{
	"/upload/*/download/builds/*":         SourceType_DownloadBuild,
	"/download-key/*/download/*/builds/*": SourceType_KeyDownloadBuild,
	"/wharf/builds/*/files/*/download":    SourceType_WharfDownloadBuild,
}

func ObtainSource(itchClient *itchio.Client, itchPath string) (*Source, error) {
	var matches bool
	var err error

	for pattern, sourceType := range patterns {
		matches, err = path.Match(pattern, itchPath)
		if err != nil {
			return nil, err
		}

		if matches {
			return &Source{
				Type:       sourceType,
				ItchClient: itchClient,
				Path:       itchPath,
			}, nil
		}
	}

	return nil, fmt.Errorf("unrecognized itchfs pattern: %s", itchPath)
}

func (s *Source) makeGetURL() (httpfile.GetURLFunc, error) {
	tokens := strings.Split(s.Path, "/")

	var getter httpfile.GetURLFunc

	switch s.Type {
	case SourceType_DownloadBuild:
		uploadID, err := strconv.ParseInt(tokens[2], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildID, err := strconv.ParseInt(tokens[5], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.DownloadUploadBuild(uploadID, buildID)
			if err != nil {
				return "", err
			}

			return r.Archive.URL, nil
		}
	case SourceType_KeyDownloadBuild:
		downloadKey := tokens[2]

		uploadID, err := strconv.ParseInt(tokens[4], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildID, err := strconv.ParseInt(tokens[6], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.DownloadUploadBuildWithKey(downloadKey, uploadID, buildID)
			if err != nil {
				return "", err
			}

			return r.Archive.URL, nil
		}
	case SourceType_WharfDownloadBuild:
		buildID, err := strconv.ParseInt(tokens[3], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildFileID, err := strconv.ParseInt(tokens[5], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.GetBuildFileDownloadURL(buildID, buildFileID)
			if err != nil {
				return "", err
			}

			return r.URL, nil
		}
	default:
		return nil, fmt.Errorf("unsupported source type: %d", s.Type)
	}

	return getter, nil
}
{'fraction_non_alphanumeric': 0.12372604973501834, 'fraction_numerical': 0.012433754586220954, 'mean_word_length': 3.4894784995425434, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '184723', 'n_tokens_mistral': 1927, 'n_tokens_neox': 1690, 'n_words': 440}