<commit_before>/*
 * gomacro - A Go interpreter with Lisp-like macros
 *
 * Copyright (C) 2017-2019 <NAME>
 *
 *     This Source Code Form is subject to the terms of the Mozilla Public
 *     License, v. 2.0. If a copy of the MPL was not distributed with this
 *     file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 *
 * importer.go
 *
 *  Created on Feb 27, 2017
 *      Author <NAME>
 */

package genimport

import (
	"bytes"
	"fmt"
	"go/build"
	"go/types"
	"io/ioutil"
	"os"
	r "reflect"

	"github.com/cosmos72/gomacro/base/output"
	"github.com/cosmos72/gomacro/base/paths"
	"github.com/cosmos72/gomacro/base/reflect"
	"github.com/cosmos72/gomacro/imports"
)

type ImportMode int

const (
	// ImBuiltin import mechanism is:
	// 1. write a file $GOPATH/src/github.com/cosmos72/gomacro/imports/$PKGPATH.go containing a single func init()
	//    i.e. *inside* gomacro sources
	// 2. tell the user to recompile gomacro
	ImBuiltin ImportMode = iota

	// ImThirdParty import mechanism is the same as ImBuiltin, except that files are created in a thirdparty/ subdirectory:
	// 1. write a file $GOPATH/src/github.com/cosmos72/gomacro/imports/thirdparty/$PKGPATH.go containing a single func init()
	//    i.e. *inside* gomacro sources
	// 2. tell the user to recompile gomacro
	ImThirdParty

	// ImInception import mechanism is:
	// 1. write a file $GOPATH/src/$PKGPATH/x_package.go containing a single func init()
	//    i.e. *inside* the package to be imported
	// 2. tell the user to recompile $PKGPATH
	ImInception

	// ImPlugin import mechanism is:
	// 1. write a file $GOPATH/src/gomacro.imports/$PKGPATH/$PKGNAME.go containing a var Packages map[string]Package
	//    and a single func init() to populate it
	// 2. invoke "go build -buildmode=plugin" on the file to create a shared library
	// 3. load such shared library with plugin.Open().Lookup("Packages")
	ImPlugin
)

type PackageName = imports.PackageName // package default name, or package alias

type PackageRef struct {
	imports.Package
	Path string
}

func (ref *PackageRef) DefaultName() PackageName {
	return ref.Package.DefaultName(ref.Path)
}

func (ref *PackageRef) String() string {
	return fmt.Sprintf("{%s %q, %d binds, %d types}", ref.DefaultName(), ref.Path, len(ref.Binds), len(ref.Types))
}

type Importer struct {
	srcDir     string
	mode       types.ImportMode
	PluginOpen r.Value // = reflect.ValueOf(plugin.Open)
	output     *Output
}

func DefaultImporter(o *Output) *Importer {
	return &Importer{output: o}
}

func (imp *Importer) havePluginOpen() bool {
	if !imp.PluginOpen.IsValid() {
		imp.PluginOpen = imports.Packages["plugin"].Binds["Open"]
		if !imp.PluginOpen.IsValid() {
			imp.PluginOpen = reflect.NoneR // cache the failure
		}
	}
	return imp.PluginOpen != reflect.NoneR
}

// LookupPackage returns a package if already present in cache
func LookupPackage(alias PackageName, pkgpath string) *PackageRef {
	pkg, found := imports.Packages[pkgpath]
	if !found {
		return nil
	}
	if len(pkg.Name) == 0 {
		// missing pkg.Name, initialize it
		pkg.DefaultName(pkgpath)
		imports.Packages[pkgpath] = pkg
	}
	if len(alias) == 0 {
		// import "foo" => get alias from package name
		alias = pkg.DefaultName(pkgpath)
	}
	return &PackageRef{Package: pkg, Path: pkgpath}
}

func (imp *Importer) wrapImportError(pkgpath string, enableModule bool, err error) output.RuntimeError {
	if rerr, ok := err.(output.RuntimeError); ok {
		return rerr
	}
	if enableModule {
		return imp.output.MakeRuntimeError("error loading package %q metadata: %v", pkgpath, err)
	}
	return imp.output.MakeRuntimeError(
		"error loading package %q metadata, maybe you need to download (go get), compile (go build) and install (go install) it? %v",
		pkgpath, err)
}

func (imp *Importer) ImportPackage(alias PackageName, pkgpath string, enableModule bool) *PackageRef {
	refs, err := imp.ImportPackagesOrError(map[string]PackageName{pkgpath: alias}, enableModule)
	if err != nil {
		panic(err)
	}
	return refs[pkgpath]
}

func (imp *Importer) ImportPackagesOrError(pkgpaths map[string]PackageName, enableModule bool) (map[string]*PackageRef, error) {
	refs := make(map[string]*PackageRef)
	for pkgpath, alias := range pkgpaths {
		ref, err := imp.importPackageOrError(alias, pkgpath, enableModule)
		if err != nil {
			return nil, err
		}
		refs[pkgpath] = ref
	}
	return refs, nil
}

func (imp *Importer) importPackageOrError(alias PackageName, pkgpath string, enableModule bool) (*PackageRef, error) {
	ref := LookupPackage(alias, pkgpath)
	if ref != nil {
		return ref, nil
	}
	paths.GetImportsSrcDir() // warns if GOPATH or paths.ImportsDir may be wrong

	o := imp.output
	gpkg, err := imp.Load(pkgpath, enableModule) // loads names and types, not the values!
	if err != nil {
		return nil, imp.wrapImportError(pkgpath, enableModule, err)
	}
	var mode ImportMode
	switch alias {
	case "_b":
		mode = ImBuiltin
	case "_i":
		mode = ImInception
	case "_3":
		mode = ImThirdParty
	default:
		if len(alias) == 0 {
			alias = PackageName(gpkg.Name())
		}
		if imp.havePluginOpen() {
			mode = ImPlugin
		} else {
			mode = ImThirdParty
		}
	}
	file := createImportFile(imp.output, pkgpath, gpkg, mode, enableModule)
	ref = &PackageRef{Path: pkgpath}
	if len(file) == 0 || mode != ImPlugin {
		// either the package exports nothing, or user must rebuild gomacro.
		// in both cases, still cache it to avoid recreating the file.
		imports.Packages[pkgpath] = ref.Package
		return ref, nil
	}
	soname := compilePlugin(o, file, enableModule, o.Stdout, o.Stderr)
	ipkgs := imp.loadPluginSymbol(soname, "Packages")
	pkgs := *ipkgs.(*map[string]imports.PackageUnderlying)

	// cache *all* packages found for future use
	imports.Packages.Merge(pkgs)

	// but return only requested one
	pkg, found := imports.Packages[pkgpath]
	if !found {
		return nil, imp.output.MakeRuntimeError(
			"error loading package %q: the compiled plugin %q does not contain it! internal error? %v",
			pkgpath, soname)
	}
	ref.Package = pkg
	return ref, nil
}

func createImportFile(o *Output, pkgpath string, pkg *types.Package, mode ImportMode, enableModule bool) string {
	dir := computeImportDir(o, pkgpath, mode)
	if mode == ImPlugin {
		createDir(o, dir)
		removeAllFilesInDirExcept(o, dir, []string{"go.mod", "go.sum"})<commit_msg>continue adding support to load multiple packages at once<commit_after>/*
 * gomacro - A Go interpreter with Lisp-like macros
 *
 * Copyright (C) 2017-2019 <NAME>
 *
 *     This Source Code Form is subject to the terms of the Mozilla Public
 *     License, v. 2.0. If a copy of the MPL was not distributed with this
 *     file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 *
 * importer.go
 *
 *  Created on Feb 27, 2017
 *      Author <NAME>
 */

package genimport

import (
	"bytes"
	"fmt"
	"go/build"
	"go/types"
	"io/ioutil"
	"os"
	r "reflect"

	"github.com/cosmos72/gomacro/base/output"
	"github.com/cosmos72/gomacro/base/paths"
	"github.com/cosmos72/gomacro/base/reflect"
	"github.com/cosmos72/gomacro/imports"
)

type ImportMode int

const (
	// ImBuiltin import mechanism is:
	// 1. write a file $GOPATH/src/github.com/cosmos72/gomacro/imports/$PKGPATH.go containing a single func init()
	//    i.e. *inside* gomacro sources
	// 2. tell the user to recompile gomacro
	ImBuiltin ImportMode = iota

	// ImThirdParty import mechanism is the same as ImBuiltin, except that files are created in a thirdparty/ subdirectory:
	// 1. write a file $GOPATH/src/github.com/cosmos72/gomacro/imports/thirdparty/$PKGPATH.go containing a single func init()
	//    i.e. *inside* gomacro sources
	// 2. tell the user to recompile gomacro
	ImThirdParty

	// ImInception import mechanism is:
	// 1. write a file $GOPATH/src/$PKGPATH/x_package.go containing a single func init()
	//    i.e. *inside* the package to be imported
	// 2. tell the user to recompile $PKGPATH
	ImInception

	// ImPlugin import mechanism is:
	// 1. write a file $GOPATH/src/gomacro.imports/$PKGPATH/$PKGNAME.go containing a var Packages map[string]Package
	//    and a single func init() to populate it
	// 2. invoke "go build -buildmode=plugin" on the file to create a shared library
	// 3. load such shared library with plugin.Open().Lookup("Packages")
	ImPlugin
)

func (mode ImportMode) String() string {
	switch mode {
	case ImBuiltin:
		return "Builtin"
	case ImThirdParty:
		return "ThirdParty"
	case ImInception:
		return "Inception"
	case ImPlugin:
		return "Plugin"
	default:
		return fmt.Sprintf("ImportMode(%d)", int(mode))
	}
}

type PackageName = imports.PackageName // package default name, or package alias

type PackageRef struct {
	imports.Package
	Path string
}

func (ref *PackageRef) DefaultName() PackageName {
	return ref.Package.DefaultName(ref.Path)
}

func (ref *PackageRef) String() string {
	return fmt.Sprintf("{%s %q, %d binds, %d types}", ref.DefaultName(), ref.Path, len(ref.Binds), len(ref.Types))
}

type Importer struct {
	srcDir     string
	mode       types.ImportMode
	PluginOpen r.Value // = reflect.ValueOf(plugin.Open)
	output     *Output
}

func DefaultImporter(o *Output) *Importer {
	return &Importer{output: o}
}

func (imp *Importer) havePluginOpen() bool {
	if !imp.PluginOpen.IsValid() {
		imp.PluginOpen = imports.Packages["plugin"].Binds["Open"]
		if !imp.PluginOpen.IsValid() {
			imp.PluginOpen = reflect.NoneR // cache the failure
		}
	}
	return imp.PluginOpen != reflect.NoneR
}

// LookupPackage returns a package if already present in cache
func LookupPackage(alias PackageName, pkgpath string) *PackageRef {
	pkg, found := imports.Packages[pkgpath]
	if !found {
		return nil
	}
	if len(pkg.Name) == 0 {
		// missing pkg.Name, initialize it
		_ = pkg.DefaultName(pkgpath)
		imports.Packages[pkgpath] = pkg
	}
	if len(alias) == 0 {
		// import "foo" => get alias from package name
		alias = pkg.DefaultName(pkgpath)
	}
	return &PackageRef{Package: pkg, Path: pkgpath}
}

func (imp *Importer) wrapImportError(pkgpath string, enableModule bool, err error) output.RuntimeError {
	if rerr, ok := err.(output.RuntimeError); ok {
		return rerr
	}
	if enableModule {
		return imp.output.MakeRuntimeError("error loading package %q metadata: %v", pkgpath, err)
	}
	return imp.output.MakeRuntimeError(
		"error loading package %q metadata, maybe you need to download (go get), compile (go build) and install (go install) it? %v",
		pkgpath, err)
}

func (imp *Importer) ImportPackage(alias PackageName, pkgpath string, enableModule bool) *PackageRef {
	refs, err := imp.ImportPackagesOrError(map[string]PackageName{pkgpath: alias}, enableModule)
	if err != nil {
		panic(err)
	}
	return refs[pkgpath]
}

func (imp *Importer) ImportPackagesOrError(pkgpaths map[string]PackageName, enableModule bool) (map[string]*PackageRef, error) {
	paths.GetImportsSrcDir() // warns if GOPATH or paths.ImportsDir may be wrong

	refs := make(map[string]*PackageRef)
	for pkgpath, alias := range pkgpaths {
		ref := LookupPackage(alias, pkgpath)
		if ref != nil {
			// found in package cache, no need to compile and load this package
			refs[pkgpath] = ref
			delete(pkgpaths, pkgpath)
		}
	}
	if len(pkgpaths) != 0 {
		mode := imp.importModeFor(pkgpaths)
		for pkgpath, alias := range pkgpaths {
			ref, err := imp.importPackageOrError(alias, pkgpath, mode, enableModule)
			if err != nil {
				return nil, err
			}
			refs[pkgpath] = ref
		}
	}
	return refs, nil
}

func (imp *Importer) importModeFor(pkgpaths map[string]PackageName) ImportMode {
	var curr ImportMode
	havemode := false
	for _, alias := range pkgpaths {
		var mode ImportMode
		switch alias {
		case "_b":
			mode = ImBuiltin
		case "_i":
			mode = ImInception
		case "_3":
			mode = ImThirdParty
		default:
			if imp.havePluginOpen() {
				mode = ImPlugin
			} else {
				mode = ImThirdParty
			}
		}
		highest := mode
		if havemode && mode != curr {
			if highest < curr {
				highest = curr
			}
			imp.output.Warnf("attempt to import multiple packages with conflicting modes %v and %v - numerically higher mode %v will be used",
				curr, mode, highest)
		}
		curr = highest
		havemode = true
	}
	return curr
}

func (imp *Importer) importPackageOrError(alias PackageName, pkgpath string, mode ImportMode, enableModule bool) (*PackageRef, error) {

	o := imp.output
	gpkg, err := imp.Load(pkgpath, enableModule) // loads names and types, not the values!
	if err != nil {
		return nil, imp.wrapImportError(pkgpath, enableModule, err)
	}
	if len(alias) == 0 {
		alias = PackageName(gpkg.Name())
	}
	file := createImportFile(imp.output, pkgpath, gpkg, mode, enableModule)
	ref := &PackageRef{Path: pkgpath}
	if len(file) == 0 || mode != ImPlugin {
		// either the package exports nothing, or user must rebuild gomacro.
		// in both cases, still cache it to avoid recreating the file.
		imports.Packages[pkgpath] = ref.Package
		return ref, nil
	}
	soname := compilePlugin(o, file, enableModule, o.Stdout, o.Stderr)
	ipkgs := imp.loadPluginSymbol(soname, "Packages")
	pkgs := *ipkgs.(*map[string]imports.PackageUnderlying)

	// cache *all* packages found for future use
	imports.Packages.Merge(pkgs)

	// but return only requested one
	pkg, found := imports.Packages[pkgpath]
	if !found {
		return nil, imp.output.MakeRuntimeError(
			"error loading package %q: the compiled plugin %q does not contain it! internal error? %v",
			pkgpath, soname)
	}
	ref.Package = pkg
	return ref, nil
}

func createImportFile(o *Output, pkgpath string, pkg *types.Package, mode ImportMode, enableModule bool) string {
	dir := computeImportDir(o, pkgpath, mode)
	if mode == ImPlugin {
		createDir(o, dir)
		removeAllFilesInDirExcept(o, dir, []string{"go.mod", "go.sum"})
{'fraction_non_alphanumeric': 0.11714958082070893, 'fraction_numerical': 0.00654508015884689, 'mean_word_length': 3.87594119756185, 'pattern_counts': {'":': 6, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7360078', 'n_tokens_mistral': 4887, 'n_tokens_neox': 4553, 'n_words': 1397}