<commit_before>package vsolver

import (
	"bytes"
	"fmt"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/Masterminds/vcs"
	"github.com/termie/go-shutil"
)

type projectManager struct {
	// The identifier of the project. At this level, corresponds to the
	// '$GOPATH/src'-relative path, *and* the network name.
	n ProjectName

	// build.Context to use in any analysis, and to pass to the analyzer
	ctx build.Context

	// Top-level project vendor dir
	vendordir string

	// Object for the cache repository
	crepo *repo

	// Indicates the extent to which we have searched for, and verified, the
	// existence of the project/repo.
	ex existence

	// Analyzer, injected by way of the SourceManager and originally from the
	// sm's creator
	an ProjectAnalyzer

	// Whether the cache has the latest info on versions
	cvsync bool

	// The project metadata cache. This is persisted to disk, for reuse across
	// solver runs.
	dc *projectDataCache
}

type existence struct {
	// The existence levels for which a search/check has been performed
	s projectExistence

	// The existence levels verified to be present through searching
	f projectExistence
}

// TODO figure out shape of versions, then implement marshaling/unmarshaling
type projectDataCache struct {
	Version string                   `json:"version"` // TODO use this
	Infos   map[Revision]projectInfo `json:"infos"`
	VMap    map[Version]Revision     `json:"vmap"`
	RMap    map[Revision][]Version   `json:"rmap"`
}

// projectInfo holds manifest and lock
type projectInfo struct {
	Manifest
	Lock
}

type repo struct {
	// Path to the root of the default working copy (NOT the repo itself)
	rpath string

	// Mutex controlling general access to the repo
	mut sync.RWMutex

	// Object for direct repo interaction
	r vcs.Repo

	// Whether or not the cache repo is in sync (think dvcs) with upstream
	synced bool
}

func (pm *projectManager) GetInfoAt(v Version) (Manifest, Lock, error) {
	if err := pm.ensureCacheExistence(); err != nil {
		return nil, nil, err
	}

	if r, exists := pm.dc.VMap[v]; exists {
		if pi, exists := pm.dc.Infos[r]; exists {
			return pi.Manifest, pi.Lock, nil
		}
	}

	pm.crepo.mut.Lock()
	var err error
	if !pm.crepo.synced {
		err = pm.crepo.r.Update()
		if err != nil {
			return nil, nil, fmt.Errorf("Could not fetch latest updates into repository")
		}
		pm.crepo.synced = true
	}

	// Always prefer a rev, if it's available
	if pv, ok := v.(PairedVersion); ok {
		err = pm.crepo.r.UpdateVersion(pv.Underlying().String())
	} else {
		err = pm.crepo.r.UpdateVersion(v.String())
	}
	pm.crepo.mut.Unlock()
	if err != nil {
		// TODO More-er proper-er error
		panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", pm.n, v.String(), err))
	}

	pm.crepo.mut.RLock()
	m, l, err := pm.an.GetInfo(pm.ctx, pm.n)
	// TODO cache results
	pm.crepo.mut.RUnlock()

	if err == nil {
		if l != nil {
			l = prepLock(l)
		}

		// If m is nil, prepManifest will provide an empty one.
		pi := projectInfo{
			Manifest: prepManifest(m, pm.n),
			Lock:     l,
		}

		if r, exists := pm.dc.VMap[v]; exists {
			pm.dc.Infos[r] = pi
		}

		return pi.Manifest, pi.Lock, nil
	}

	return nil, nil, err
}

func (pm *projectManager) ListPackages(v Version) (PackageTree, error) {
	var err error
	if err = pm.ensureCacheExistence(); err != nil {
		return PackageTree{}, err
	}

	pm.crepo.mut.Lock()
	// Check out the desired version for analysis
	if pv, ok := v.(PairedVersion); ok {
		// Always prefer a rev, if it's available
		err = pm.crepo.r.UpdateVersion(pv.Underlying().String())
	} else {
		// If we don't have a rev, ensure the repo is up to date, otherwise we
		// could have a desync issue
		if !pm.crepo.synced {
			err = pm.crepo.r.Update()
			if err != nil {
				return PackageTree{}, fmt.Errorf("Could not fetch latest updates into repository")
			}
			pm.crepo.synced = true
		}
		err = pm.crepo.r.UpdateVersion(v.String())
	}

	ex, err := listPackages(filepath.Join(pm.ctx.GOPATH, "src", string(pm.n)), string(pm.n))
	pm.crepo.mut.Unlock()

	return ex, err
}

func (pm *projectManager) ensureCacheExistence() error {
	// Technically, methods could could attempt to return straight from the
	// metadata cache even if the repo cache doesn't exist on disk. But that
	// would allow weird state inconsistencies (cache exists, but no repo...how
	// does that even happen?) that it'd be better to just not allow so that we
	// don't have to think about it elsewhere
	if !pm.CheckExistence(existsInCache) {
		if pm.CheckExistence(existsUpstream) {
			err := pm.crepo.r.Get()
			if err != nil {
				return fmt.Errorf("Failed to create repository cache for %s", pm.n)
			}
			pm.ex.s |= existsInCache
			pm.ex.f |= existsInCache
		} else {
			return fmt.Errorf("Project repository cache for %s does not exist", pm.n)
		}
	}

	return nil
}

func (pm *projectManager) ListVersions() (vlist []Version, err error) {
	if !pm.cvsync {
		// This check only guarantees that the upstream exists, not the cache
		pm.ex.s |= existsUpstream
		vpairs, exbits, err := pm.crepo.getCurrentVersionPairs()
		// But it *may* also check the local existence<commit_msg>Small internal fixups to projectManager<commit_after>package vsolver

import (
	"bytes"
	"fmt"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/Masterminds/vcs"
	"github.com/termie/go-shutil"
)

type projectManager struct {
	// The identifier of the project. At this level, corresponds to the
	// '$GOPATH/src'-relative path, *and* the network name.
	n ProjectName

	// build.Context to use in any analysis, and to pass to the analyzer
	ctx build.Context

	// Top-level project vendor dir
	vendordir string

	// Object for the cache repository
	crepo *repo

	// Indicates the extent to which we have searched for, and verified, the
	// existence of the project/repo.
	ex existence

	// Analyzer, injected by way of the SourceManager and originally from the
	// sm's creator
	an ProjectAnalyzer

	// Whether the cache has the latest info on versions
	cvsync bool

	// The project metadata cache. This is persisted to disk, for reuse across
	// solver runs.
	// TODO protect with mutex
	dc *projectDataCache
}

type existence struct {
	// The existence levels for which a search/check has been performed
	s projectExistence

	// The existence levels verified to be present through searching
	f projectExistence
}

// TODO figure out shape of versions, then implement marshaling/unmarshaling
type projectDataCache struct {
	Version string                   `json:"version"` // TODO use this
	Infos   map[Revision]projectInfo `json:"infos"`
	VMap    map[Version]Revision     `json:"vmap"`
	RMap    map[Revision][]Version   `json:"rmap"`
}

// projectInfo holds manifest and lock
type projectInfo struct {
	Manifest
	Lock
}

type repo struct {
	// Path to the root of the default working copy (NOT the repo itself)
	rpath string

	// Mutex controlling general access to the repo
	mut sync.RWMutex

	// Object for direct repo interaction
	r vcs.Repo

	// Whether or not the cache repo is in sync (think dvcs) with upstream
	synced bool
}

func (pm *projectManager) GetInfoAt(v Version) (Manifest, Lock, error) {
	if err := pm.ensureCacheExistence(); err != nil {
		return nil, nil, err
	}

	if r, exists := pm.dc.VMap[v]; exists {
		if pi, exists := pm.dc.Infos[r]; exists {
			return pi.Manifest, pi.Lock, nil
		}
	}

	pm.crepo.mut.Lock()
	var err error
	if !pm.crepo.synced {
		err = pm.crepo.r.Update()
		if err != nil {
			return nil, nil, fmt.Errorf("Could not fetch latest updates into repository")
		}
		pm.crepo.synced = true
	}

	// Always prefer a rev, if it's available
	if pv, ok := v.(PairedVersion); ok {
		err = pm.crepo.r.UpdateVersion(pv.Underlying().String())
	} else {
		err = pm.crepo.r.UpdateVersion(v.String())
	}
	pm.crepo.mut.Unlock()
	if err != nil {
		// TODO More-er proper-er error
		panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", pm.n, v.String(), err))
	}

	pm.crepo.mut.RLock()
	m, l, err := pm.an.GetInfo(pm.ctx, pm.n)
	// TODO cache results
	pm.crepo.mut.RUnlock()

	if err == nil {
		if l != nil {
			l = prepLock(l)
		}

		// If m is nil, prepManifest will provide an empty one.
		pi := projectInfo{
			Manifest: prepManifest(m, pm.n),
			Lock:     l,
		}

		// TODO this just clobbers all over and ignores the paired/unpaired
		// distinction; serious fix is needed
		if r, exists := pm.dc.VMap[v]; exists {
			pm.dc.Infos[r] = pi
		}

		return pi.Manifest, pi.Lock, nil
	}

	return nil, nil, err
}

func (pm *projectManager) ListPackages(v Version) (PackageTree, error) {
	var err error
	if err = pm.ensureCacheExistence(); err != nil {
		return PackageTree{}, err
	}

	pm.crepo.mut.Lock()
	// Check out the desired version for analysis
	if pv, ok := v.(PairedVersion); ok {
		// Always prefer a rev, if it's available
		err = pm.crepo.r.UpdateVersion(pv.Underlying().String())
	} else {
		// If we don't have a rev, ensure the repo is up to date, otherwise we
		// could have a desync issue
		if !pm.crepo.synced {
			err = pm.crepo.r.Update()
			if err != nil {
				return PackageTree{}, fmt.Errorf("Could not fetch latest updates into repository")
			}
			pm.crepo.synced = true
		}
		err = pm.crepo.r.UpdateVersion(v.String())
	}

	ex, err := listPackages(filepath.Join(pm.ctx.GOPATH, "src", string(pm.n)), string(pm.n))
	pm.crepo.mut.Unlock()

	return ex, err
}

func (pm *projectManager) ensureCacheExistence() error {
	// Technically, methods could could attempt to return straight from the
	// metadata cache even if the repo cache doesn't exist on disk. But that
	// would allow weird state inconsistencies (cache exists, but no repo...how
	// does that even happen?) that it'd be better to just not allow so that we
	// don't have to think about it elsewhere
	if !pm.CheckExistence(existsInCache) {
		if pm.CheckExistence(existsUpstream) {
			pm.crepo.mut.Lock()
			err := pm.crepo.r.Get()
			pm.crepo.mut.Unlock()

			if err != nil {
				return fmt.Errorf("failed to create repository cache for %s", pm.n)
			}
			pm.ex.s |= existsInCache
			pm.ex.f |= existsInCache
		} else {
			return fmt.Errorf("project %s does not exist upstream", pm.n)
		}
	}

	return nil
}

func (pm *projectManager) ListVersions() (vlist []Version, err error) {
	if !pm.cvsync {
		// This check only guarantees that the upstream exists, not the cache
		pm.ex.s |= existsUpstream
		vpairs, exbits, err := pm.crepo.getCurrentVersionPairs()
		// But it *may* also check the local existence
{'fraction_non_alphanumeric': 0.11637399145704794, 'fraction_numerical': 0.0, 'mean_word_length': 3.238133547868061, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1633317', 'n_tokens_mistral': 3721, 'n_tokens_neox': 3489, 'n_words': 1214}