<commit_before>package main

import (
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/cancel"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"

	log "gopkg.in/inconshreveable/log15.v2"
)

// Simplestream cache
type imageStreamCacheEntry struct {
	Aliases      []api.ImageAliasesEntry `yaml:"aliases"`
	Certificate  string                  `yaml:"certificate"`
	Fingerprints []string                `yaml:"fingerprints"`

	expiry time.Time
	remote lxd.ImageServer
}

var imageStreamCache = map[string]*imageStreamCacheEntry{}
var imageStreamCacheLock sync.Mutex

var imagesDownloading = map[string]chan bool{}
var imagesDownloadingLock sync.Mutex

func imageSaveStreamCache() error {
	data, err := yaml.Marshal(&imageStreamCache)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(shared.CachePath("simplestreams.yaml"), data, 0600)
	if err != nil {
		return err
	}

	return nil
}

func imageLoadStreamCache(d *Daemon) error {
	imageStreamCacheLock.Lock()
	defer imageStreamCacheLock.Unlock()

	if !shared.PathExists(shared.CachePath("simplestreams.yaml")) {
		return nil
	}

	content, err := ioutil.ReadFile(shared.CachePath("simplestreams.yaml"))
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(content, imageStreamCache)
	if err != nil {
		return err
	}

	for url, entry := range imageStreamCache {
		if entry.remote == nil {
			remote, err := lxd.ConnectSimpleStreams(url, &lxd.ConnectionArgs{
				TLSServerCert: entry.Certificate,
				UserAgent:     version.UserAgent,
				Proxy:         d.proxy,
			})
			if err != nil {
				continue
			}

			entry.remote = remote
		}
	}

	return nil
}

// ImageDownload resolves the image fingerprint and if not in the database, downloads it
func (d *Daemon) ImageDownload(op *operation, server string, protocol string, certificate string, secret string, alias string, forContainer bool, autoUpdate bool, storagePool string, preferCached bool) (*api.Image, error) {
	var err error
	var ctxMap log.Ctx

	var remote lxd.ImageServer
	var info *api.Image

	// Default protocol is LXD
	if protocol == "" {
		protocol = "lxd"
	}

	// Default the fingerprint to the alias string we received
	fp := alias

	// Attempt to resolve the alias
	if protocol == "simplestreams" {
		imageStreamCacheLock.Lock()
		entry, _ := imageStreamCache[server]
		if entry == nil || entry.expiry.Before(time.Now()) {
			// Add a new entry to the cache
			refresh := func() (*imageStreamCacheEntry, error) {
				// Setup simplestreams client
				remote, err = lxd.ConnectSimpleStreams(server, &lxd.ConnectionArgs{
					TLSServerCert: certificate,
					UserAgent:     version.UserAgent,
					Proxy:         d.proxy,
				})
				if err != nil {
					return nil, err
				}

				// Get all aliases
				aliases, err := remote.GetImageAliases()
				if err != nil {
					return nil, err
				}

				// Get all fingerprints
				images, err := remote.GetImages()
				if err != nil {
					return nil, err
				}

				fingerprints := []string{}
				for _, image := range images {
					fingerprints = append(fingerprints, image.Fingerprint)
				}

				// Generate cache entry
				entry = &imageStreamCacheEntry{remote: remote, Aliases: aliases, Certificate: certificate, Fingerprints: fingerprints, expiry: time.Now().Add(time.Hour)}
				imageStreamCache[server] = entry
				imageSaveStreamCache()

				return entry, nil
			}

			newEntry, err := refresh()
			if err == nil {
				// Cache refreshed
				entry = newEntry
			} else if entry != nil {
				// Failed to fetch entry but existing cache
				logger.Warn("Unable to refresh cache, using stale entry", log.Ctx{"server": server})
				entry.expiry = time.Now().Add(time.Hour)
			} else {
				// Failed to fetch entry and nothing in cache
				imageStreamCacheLock.Unlock()
				return nil, err
			}
		} else {
			// use the existing entry
			logger.Debug("Using SimpleStreams cache entry", log.Ctx{"server": server, "expiry": entry.expiry})
			remote = entry.remote
		}
		imageStreamCacheLock.Unlock()

		// Look for a matching alias
		for _, entry := range entry.Aliases {
			if entry.Name != fp {
				continue
			}

			fp = entry.Target
			break
		}

		// Expand partial fingerprints
		matches := []string{}
		for _, entry := range entry.Fingerprints {
			if strings.HasPrefix(entry, fp) {
				matches = append(matches, entry)
			}
		}

		if len(matches) == 1 {
			fp = matches[0]
		} else if len(matches) > 1 {
			return nil, fmt.Errorf("Provided partial image fingerprint matches more than one image")
		}
	} else if protocol == "lxd" {
		// Setup LXD client<commit_msg>lxd/images: Clear error for image not found<commit_after>package main

import (
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/cancel"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"

	log "gopkg.in/inconshreveable/log15.v2"
)

// Simplestream cache
type imageStreamCacheEntry struct {
	Aliases      []api.ImageAliasesEntry `yaml:"aliases"`
	Certificate  string                  `yaml:"certificate"`
	Fingerprints []string                `yaml:"fingerprints"`

	expiry time.Time
	remote lxd.ImageServer
}

var imageStreamCache = map[string]*imageStreamCacheEntry{}
var imageStreamCacheLock sync.Mutex

var imagesDownloading = map[string]chan bool{}
var imagesDownloadingLock sync.Mutex

func imageSaveStreamCache() error {
	data, err := yaml.Marshal(&imageStreamCache)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(shared.CachePath("simplestreams.yaml"), data, 0600)
	if err != nil {
		return err
	}

	return nil
}

func imageLoadStreamCache(d *Daemon) error {
	imageStreamCacheLock.Lock()
	defer imageStreamCacheLock.Unlock()

	if !shared.PathExists(shared.CachePath("simplestreams.yaml")) {
		return nil
	}

	content, err := ioutil.ReadFile(shared.CachePath("simplestreams.yaml"))
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(content, imageStreamCache)
	if err != nil {
		return err
	}

	for url, entry := range imageStreamCache {
		if entry.remote == nil {
			remote, err := lxd.ConnectSimpleStreams(url, &lxd.ConnectionArgs{
				TLSServerCert: entry.Certificate,
				UserAgent:     version.UserAgent,
				Proxy:         d.proxy,
			})
			if err != nil {
				continue
			}

			entry.remote = remote
		}
	}

	return nil
}

// ImageDownload resolves the image fingerprint and if not in the database, downloads it
func (d *Daemon) ImageDownload(op *operation, server string, protocol string, certificate string, secret string, alias string, forContainer bool, autoUpdate bool, storagePool string, preferCached bool) (*api.Image, error) {
	var err error
	var ctxMap log.Ctx

	var remote lxd.ImageServer
	var info *api.Image

	// Default protocol is LXD
	if protocol == "" {
		protocol = "lxd"
	}

	// Default the fingerprint to the alias string we received
	fp := alias

	// Attempt to resolve the alias
	if protocol == "simplestreams" {
		imageStreamCacheLock.Lock()
		entry, _ := imageStreamCache[server]
		if entry == nil || entry.expiry.Before(time.Now()) {
			// Add a new entry to the cache
			refresh := func() (*imageStreamCacheEntry, error) {
				// Setup simplestreams client
				remote, err = lxd.ConnectSimpleStreams(server, &lxd.ConnectionArgs{
					TLSServerCert: certificate,
					UserAgent:     version.UserAgent,
					Proxy:         d.proxy,
				})
				if err != nil {
					return nil, err
				}

				// Get all aliases
				aliases, err := remote.GetImageAliases()
				if err != nil {
					return nil, err
				}

				// Get all fingerprints
				images, err := remote.GetImages()
				if err != nil {
					return nil, err
				}

				fingerprints := []string{}
				for _, image := range images {
					fingerprints = append(fingerprints, image.Fingerprint)
				}

				// Generate cache entry
				entry = &imageStreamCacheEntry{remote: remote, Aliases: aliases, Certificate: certificate, Fingerprints: fingerprints, expiry: time.Now().Add(time.Hour)}
				imageStreamCache[server] = entry
				imageSaveStreamCache()

				return entry, nil
			}

			newEntry, err := refresh()
			if err == nil {
				// Cache refreshed
				entry = newEntry
			} else if entry != nil {
				// Failed to fetch entry but existing cache
				logger.Warn("Unable to refresh cache, using stale entry", log.Ctx{"server": server})
				entry.expiry = time.Now().Add(time.Hour)
			} else {
				// Failed to fetch entry and nothing in cache
				imageStreamCacheLock.Unlock()
				return nil, err
			}
		} else {
			// use the existing entry
			logger.Debug("Using SimpleStreams cache entry", log.Ctx{"server": server, "expiry": entry.expiry})
			remote = entry.remote
		}
		imageStreamCacheLock.Unlock()

		// Look for a matching alias
		for _, entry := range entry.Aliases {
			if entry.Name != fp {
				continue
			}

			fp = entry.Target
			break
		}

		// Expand partial fingerprints
		matches := []string{}
		for _, entry := range entry.Fingerprints {
			if strings.HasPrefix(entry, fp) {
				matches = append(matches, entry)
			}
		}

		if len(matches) == 1 {
			fp = matches[0]
		} else if len(matches) > 1 {
			return nil, fmt.Errorf("Provided partial image fingerprint matches more than one image")
		} else {
			return nil, fmt.Errorf("The requested image couldn't be found.")
		}
	} else if protocol == "lxd" {
		// Setup LXD client
{'fraction_non_alphanumeric': 0.11661687442052128, 'fraction_numerical': 0.002884516328422788, 'mean_word_length': 3.220869565217391, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7446161', 'n_tokens_mistral': 3659, 'n_tokens_neox': 3106, 'n_words': 844}