<commit_before>package main

import (
	"crypto/sha256"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/cancel"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/version"

	log "github.com/lxc/lxd/shared/log15"
)

var imagesDownloading = map[string]chan bool{}
var imagesDownloadingLock sync.Mutex

// ImageDownload resolves the image fingerprint and if not in the database, downloads it
func (d *Daemon) ImageDownload(op *operations.Operation, server string, protocol string, certificate string, secret string, alias string, imageType string, forContainer bool, autoUpdate bool, storagePool string, preferCached bool, project string, budget int64) (*api.Image, error) {
	var err error
	var ctxMap log.Ctx

	var remote lxd.ImageServer
	var info *api.Image

	// Default protocol is LXD
	if protocol == "" {
		protocol = "lxd"
	}

	// Default the fingerprint to the alias string we received
	fp := alias

	// Attempt to resolve the alias
	if shared.StringInSlice(protocol, []string{"lxd", "simplestreams"}) {
		args := &lxd.ConnectionArgs{
			TLSServerCert: certificate,
			UserAgent:     version.UserAgent,
			Proxy:         d.proxy,
			CachePath:     d.os.CacheDir,
			CacheExpiry:   time.Hour,
		}

		if protocol == "lxd" {
			// Setup LXD client
			remote, err = lxd.ConnectPublicLXD(server, args)
			if err != nil {
				return nil, err
			}
		} else {
			// Setup simplestreams client
			remote, err = lxd.ConnectSimpleStreams(server, args)
			if err != nil {
				return nil, err
			}
		}

		// For public images, handle aliases and initial metadata
		if secret == "" {
			// Look for a matching alias
			entry, _, err := remote.GetImageAliasType(imageType, fp)
			if err == nil {
				fp = entry.Target
			}

			// Expand partial fingerprints
			info, _, err = remote.GetImage(fp)
			if err != nil {
				return nil, err
			}

			fp = info.Fingerprint
		}
	}

	// If auto-update is on and we're being given the image by
	// alias, try to use a locally cached image matching the given
	// server/protocol/alias, regardless of whether it's stale or
	// not (we can assume that it will be not *too* stale since
	// auto-update is on).
	interval, err := cluster.ConfigGetInt64(d.cluster, "images.auto_update_interval")
	if err != nil {
		return nil, err
	}
	if preferCached && interval > 0 && alias != fp {
		for _, architecture := range d.os.Architectures {
			cachedFingerprint, err := d.cluster.GetCachedImageSourceFingerprint(server, protocol, alias, imageType, architecture)
			if err == nil && cachedFingerprint != fp {
				fp = cachedFingerprint
				break
			}
		}
	}

	// Check if the image already exists in this project (partial hash match)
	_, imgInfo, err := d.cluster.GetImage(project, fp, false)
	if err == db.ErrNoSuchObject {
		// Check if the image already exists in some other project.
		_, imgInfo, err = d.cluster.GetImageFromAnyProject(fp)
		if err == nil {
			// We just need to insert the database data, no actual download necessary.
			err = d.cluster.CreateImage(
				project, imgInfo.Fingerprint, imgInfo.Filename, imgInfo.Size, false,
				imgInfo.AutoUpdate, imgInfo.Architecture, imgInfo.CreatedAt, imgInfo.ExpiresAt,
				imgInfo.Properties, imgInfo.Type)
			if err != nil {
				return nil, err
			}

			var id int
			id, imgInfo, err = d.cluster.GetImage(project, fp, false)
			if err != nil {
				return nil, err
			}
			err = d.cluster.CreateImageSource(id, server, protocol, certificate, alias)
			if err != nil {
				return nil, err
			}
		}
	}

	if err == nil {
		logger.Debug("Image already exists in the db", log.Ctx{"image": fp})
		info = imgInfo

		// If not requested in a particular pool, we're done.
		if storagePool == "" {
			return info, nil<commit_msg>lxd/daemon/image: Adds logic to download image from another cluster node into ImageDownload<commit_after>package main

import (
	"crypto/sha256"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/cancel"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/version"

	log "github.com/lxc/lxd/shared/log15"
)

var imagesDownloading = map[string]chan bool{}
var imagesDownloadingLock sync.Mutex

// ImageDownload resolves the image fingerprint and if not in the database, downloads it
func (d *Daemon) ImageDownload(op *operations.Operation, server string, protocol string, certificate string, secret string, alias string, imageType string, forContainer bool, autoUpdate bool, storagePool string, preferCached bool, project string, budget int64) (*api.Image, error) {
	var err error
	var ctxMap log.Ctx

	var remote lxd.ImageServer
	var info *api.Image

	// Default protocol is LXD
	if protocol == "" {
		protocol = "lxd"
	}

	// Default the fingerprint to the alias string we received
	fp := alias

	// Attempt to resolve the alias
	if shared.StringInSlice(protocol, []string{"lxd", "simplestreams"}) {
		args := &lxd.ConnectionArgs{
			TLSServerCert: certificate,
			UserAgent:     version.UserAgent,
			Proxy:         d.proxy,
			CachePath:     d.os.CacheDir,
			CacheExpiry:   time.Hour,
		}

		if protocol == "lxd" {
			// Setup LXD client
			remote, err = lxd.ConnectPublicLXD(server, args)
			if err != nil {
				return nil, err
			}
		} else {
			// Setup simplestreams client
			remote, err = lxd.ConnectSimpleStreams(server, args)
			if err != nil {
				return nil, err
			}
		}

		// For public images, handle aliases and initial metadata
		if secret == "" {
			// Look for a matching alias
			entry, _, err := remote.GetImageAliasType(imageType, fp)
			if err == nil {
				fp = entry.Target
			}

			// Expand partial fingerprints
			info, _, err = remote.GetImage(fp)
			if err != nil {
				return nil, err
			}

			fp = info.Fingerprint
		}
	}

	// If auto-update is on and we're being given the image by
	// alias, try to use a locally cached image matching the given
	// server/protocol/alias, regardless of whether it's stale or
	// not (we can assume that it will be not *too* stale since
	// auto-update is on).
	interval, err := cluster.ConfigGetInt64(d.cluster, "images.auto_update_interval")
	if err != nil {
		return nil, err
	}
	if preferCached && interval > 0 && alias != fp {
		for _, architecture := range d.os.Architectures {
			cachedFingerprint, err := d.cluster.GetCachedImageSourceFingerprint(server, protocol, alias, imageType, architecture)
			if err == nil && cachedFingerprint != fp {
				fp = cachedFingerprint
				break
			}
		}
	}

	// Check if the image already exists in this project (partial hash match).
	_, imgInfo, err := d.cluster.GetImage(project, fp, false)
	if err == nil {
		// Check if the image is available locally or it's on another node.
		nodeAddress, err := d.State().Cluster.LocateImage(imgInfo.Fingerprint)
		if err != nil {
			return nil, errors.Wrapf(err, "Locate image %q in the cluster", imgInfo.Fingerprint)
		}

		if nodeAddress != "" {
			// The image is available from another node, let's try to import it.
			err = instanceImageTransfer(d, project, info.Fingerprint, nodeAddress)
			if err != nil {
				return nil, errors.Wrapf(err, "Failed transferring image")
			}

			// As the image record already exists in the project, just add the node ID to the image.
			err = d.cluster.AddImageToLocalNode(project, info.Fingerprint)
			if err != nil {
				return nil, errors.Wrapf(err, "Failed adding image to local node")
			}
		}
	} else if err == db.ErrNoSuchObject {
		// Check if the image already exists in some other project.
		_, imgInfo, err = d.cluster.GetImageFromAnyProject(fp)
		if err == nil {
			// Check if the image is available locally or it's on another node. Do this before creating
			// the missing DB record so we don't include ourself in the search results.
			nodeAddress, err := d.State().Cluster.LocateImage(imgInfo.Fingerprint)
			if err != nil {
				return nil, errors.Wrapf(err, "Locate image %q in the cluster", imgInfo.Fingerprint)
			}

			// We need to insert the database entry for this project, including the node ID entry.
			err = d.cluster.CreateImage(
				project, imgInfo.Fingerprint, imgInfo.Filename, imgInfo.Size, false,
				imgInfo.AutoUpdate, imgInfo.Architecture, imgInfo.CreatedAt, imgInfo.ExpiresAt,
				imgInfo.Properties, imgInfo.Type)
			if err != nil {
				return nil, err
			}

			var id int
			id, imgInfo, err = d.cluster.GetImage(project, fp, false)
			if err != nil {
				return nil, err
			}

			err = d.cluster.CreateImageSource(id, server, protocol, certificate, alias)
			if err != nil {
				return nil, err
			}

			// Transfer image if needed (after database record has been created above).
			if nodeAddress != "" {
				// The image is available from another node, let's try to import it.
				err = instanceImageTransfer(d, project, info.Fingerprint, nodeAddress)
				if err != nil {
					return nil, errors.Wrapf(err, "Failed transferring image")
				}
			}
		}
	}

	if imgInfo != nil {
		logger.Debugf("Image %q already exists in the DB", fp)
		info = imgInfo

		// If not requested in a particular pool, we're done.
		if storagePool == "" {
			return info, nil
{'fraction_non_alphanumeric': 0.11355914753423248, 'fraction_numerical': 0.002059096056831051, 'mean_word_length': 3.2755281690140845, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6160011', 'n_tokens_mistral': 3587, 'n_tokens_neox': 3177, 'n_words': 982}