<commit_before>package push

import (
	"fmt"
	"io/ioutil"
	"math"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/go-errors/errors"
	"github.com/itchio/butler/comm"
	"github.com/itchio/butler/mansion"
	itchio "github.com/itchio/go-itchio"
	"github.com/itchio/httpkit/uploader"
	"github.com/itchio/savior/seeksource"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/state"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wsync"
)

const (
	// almostThereThreshold is the amount of data left where the progress indicator isn't indicative anymore.
	// At this point, we're basically waiting for build files to be finalized.
	almostThereThreshold int64 = 10 * 1024
)

var args = struct {
	src             *string
	target          *string
	userVersion     *string
	userVersionFile *string
	fixPerms        *bool
	dereference     *bool
}{}

func Register(ctx *mansion.Context) {
	cmd := ctx.App.Command("push", "Upload a new build to itch.io. See `butler help push`.")
	args.src = cmd.Arg("src", "Directory to upload. May also be a zip archive (slower)").Required().String()
	args.target = cmd.Arg("target", "Where to push, for example 'leafo/x-moon:win-64'. Targets are of the form project:channel, where project is username/game or game_id.").Required().String()
	args.userVersion = cmd.Flag("userversion", "A user-supplied version number that you can later query builds by").String()
	args.userVersionFile = cmd.Flag("userversion-file", "A file containing a user-supplied version number that you can later query builds by").String()
	args.fixPerms = cmd.Flag("fix-permissions", "Detect Mac & Linux executables and adjust their permissions automatically").Default("true").Bool()
	args.dereference = cmd.Flag("dereference", "Dereference symlinks").Default("false").Bool()
	ctx.Register(cmd, do)
}

func do(ctx *mansion.Context) {
	go ctx.DoVersionCheck()

	// if userVersionFile specified, read from the given file
	// TODO: do utf-16 decoding here
	userVersion := *args.userVersion
	if userVersion == "" && *args.userVersionFile != "" {
		buf, err := ioutil.ReadFile(*args.userVersionFile)
		ctx.Must(err)

		userVersion = strings.TrimSpace(string(buf))
		if strings.ContainsAny(userVersion, "\r\n") {
			ctx.Must(fmt.Errorf("%s contains line breaks, refusing to use as userversion", *args.userVersionFile))
		}
	}

	ctx.Must(Do(ctx, *args.src, *args.target, userVersion, *args.fixPerms, *args.dereference))
}

func Do(ctx *mansion.Context, buildPath string, specStr string, userVersion string, fixPerms bool, dereference bool) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan walkResult)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs, fixPerms, dereference)

	spec, err := itchio.ParseSpec(specStr)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = spec.EnsureChannel()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	client, err := ctx.AuthenticateViaOauth()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	newBuildRes, err := client.CreateBuild(spec.Target, spec.Channel, userVersion)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature *pwr.SignatureInfo

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", spec.Channel)
		targetSignature = &pwr.SignatureInfo{
			Container: &tlc.Container{},
			Hashes:    make([]wsync.BlockHash, 0),
		}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", spec.Channel, parentID)
		buildFiles, err := client.ListBuildFiles(parentID)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		signatureFile := itchio.FindBuildFile(itchio.BuildFileTypeSignature, buildFiles.Files)
		if signatureFile == nil {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		signatureURL := itchio.ItchfsURL(
			parentID,
			signatureFile.ID,
			client.Key,
		)

		signatureReader, err := eos.Open(signatureURL)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		signatureSource := seeksource.FromFile(signatureReader)

		_, err = signatureSource.Resume(nil)
		if err != nil {
			return errors.Wrap(err, 0)
		}

		targetSignature, err = pwr.ReadSignature(signatureSource)
		if err != nil {
			return errors.Wrap(err, 1)
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	consumer := comm.NewStateConsumer()

	patchWriter := uploader.NewResumableUpload2(newPatchRes.File.UploadURL)
	patchWriter.SetConsumer(consumer)

	signatureWriter := uploader.NewResumableUpload2(newSignatureRes.File.UploadURL)
	signatureWriter.SetConsumer(consumer)

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.<commit_msg>push: introduce --if-changed flag, to avoid pushing if nothing's changed<commit_after>package push

import (
	"fmt"
	"io/ioutil"
	"math"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/go-errors/errors"
	"github.com/itchio/butler/comm"
	"github.com/itchio/butler/mansion"
	itchio "github.com/itchio/go-itchio"
	"github.com/itchio/httpkit/uploader"
	"github.com/itchio/savior/seeksource"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/state"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wsync"
)

const (
	// almostThereThreshold is the amount of data left where the progress indicator isn't indicative anymore.
	// At this point, we're basically waiting for build files to be finalized.
	almostThereThreshold int64 = 10 * 1024
)

var args = struct {
	src             *string
	target          *string
	userVersion     *string
	userVersionFile *string
	fixPerms        *bool
	dereference     *bool
	ifChanged       *bool
}{}

func Register(ctx *mansion.Context) {
	cmd := ctx.App.Command("push", "Upload a new build to itch.io. See `butler help push`.")
	args.src = cmd.Arg("src", "Directory to upload. May also be a zip archive (slower)").Required().String()
	args.target = cmd.Arg("target", "Where to push, for example 'leafo/x-moon:win-64'. Targets are of the form project:channel, where project is username/game or game_id.").Required().String()
	args.userVersion = cmd.Flag("userversion", "A user-supplied version number that you can later query builds by").String()
	args.userVersionFile = cmd.Flag("userversion-file", "A file containing a user-supplied version number that you can later query builds by").String()
	args.fixPerms = cmd.Flag("fix-permissions", "Detect Mac & Linux executables and adjust their permissions automatically").Default("true").Bool()
	args.dereference = cmd.Flag("dereference", "Dereference symlinks").Default("false").Bool()
	args.ifChanged = cmd.Flag("if-changed", "Don't push anything if it would be an empty patch").Default("false").Bool()
	ctx.Register(cmd, do)
}

func do(ctx *mansion.Context) {
	go ctx.DoVersionCheck()

	// if userVersionFile specified, read from the given file
	// TODO: do utf-16 decoding here
	userVersion := *args.userVersion
	if userVersion == "" && *args.userVersionFile != "" {
		buf, err := ioutil.ReadFile(*args.userVersionFile)
		ctx.Must(err)

		userVersion = strings.TrimSpace(string(buf))
		if strings.ContainsAny(userVersion, "\r\n") {
			ctx.Must(fmt.Errorf("%s contains line breaks, refusing to use as userversion", *args.userVersionFile))
		}
	}

	ctx.Must(Do(ctx, *args.src, *args.target, userVersion, *args.fixPerms, *args.dereference, *args.ifChanged))
}

func Do(ctx *mansion.Context, buildPath string, specStr string, userVersion string, fixPerms bool, dereference bool, ifChanged bool) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan walkResult)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs, fixPerms, dereference)

	spec, err := itchio.ParseSpec(specStr)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = spec.EnsureChannel()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	client, err := ctx.AuthenticateViaOauth()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	newBuildRes, err := client.CreateBuild(spec.Target, spec.Channel, userVersion)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature *pwr.SignatureInfo

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", spec.Channel)
		targetSignature = &pwr.SignatureInfo{
			Container: &tlc.Container{},
			Hashes:    make([]wsync.BlockHash, 0),
		}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", spec.Channel, parentID)
		buildFiles, err := client.ListBuildFiles(parentID)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		signatureFile := itchio.FindBuildFile(itchio.BuildFileTypeSignature, buildFiles.Files)
		if signatureFile == nil {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		signatureURL := itchio.ItchfsURL(
			parentID,
			signatureFile.ID,
			client.Key,
		)

		signatureReader, err := eos.Open(signatureURL)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		signatureSource := seeksource.FromFile(signatureReader)

		_, err = signatureSource.Resume(nil)
		if err != nil {
			return errors.Wrap(err, 0)
		}

		targetSignature, err = pwr.ReadSignature(signatureSource)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		if ifChanged {
			vctx := pwr.ValidatorContext{
				Consumer:   comm.NewStateConsumer(),
				NumWorkers: 1,
			}
			err := vctx.Validate(buildPath, targetSignature)
			if err == nil {
				comm.Statf("No changes and --if-changed used, not pushing anything")
				return nil
			}
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	consumer := comm.NewStateConsumer()

	patchWriter := uploader.NewResumableUpload2(newPatchRes.File.UploadURL)
	patchWriter.SetConsumer(consumer)

	signatureWriter := uploader.NewResumableUpload2(newSignatureRes.File.UploadURL)
	signatureWriter.SetConsumer(consumer)

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.
{'fraction_non_alphanumeric': 0.12650433809123984, 'fraction_numerical': 0.004757906521130702, 'mean_word_length': 4.328031809145129, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1497239', 'n_tokens_mistral': 3767, 'n_tokens_neox': 3483, 'n_words': 885}