<commit_before>package extractor

import (
	"fmt"
	"go/ast"
	"go/constant"
	"go/scanner"
	"go/token"
	"go/types"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/github/codeql-go/extractor/dbscheme"
	"github.com/github/codeql-go/extractor/srcarchive"
	"github.com/github/codeql-go/extractor/trap"
	"github.com/github/codeql-go/extractor/util"
	"golang.org/x/tools/go/packages"
)

// Extract extracts the packages specified by the given patterns
func Extract(patterns []string) error {
	return ExtractWithFlags(nil, patterns)
}

// ExtractWithFlags extracts the packages specified by the given patterns and build flags
func ExtractWithFlags(buildFlags []string, patterns []string) error {
	cfg := &packages.Config{
		Mode: packages.NeedName | packages.NeedFiles |
			packages.NeedCompiledGoFiles |
			packages.NeedImports | packages.NeedDeps |
			packages.NeedTypes | packages.NeedTypesSizes |
			packages.NeedTypesInfo | packages.NeedSyntax,
		BuildFlags: buildFlags,
	}
	pkgs, err := packages.Load(cfg, patterns...)

	modFlags := make([]string, 0, 1)
	for _, flag := range buildFlags {
		if strings.HasPrefix(flag, "-mod=") {
			modFlags = append(modFlags, flag)
		}
	}

	if err != nil {
		return err
	}

	if len(pkgs) == 0 {
		log.Printf("No packages found.")
	}

	extractUniverseScope()

	// a map of package path to package root directory (currently the module root or the source directory)
	pkgRoots := make(map[string]string)
	// a map of package path to source code directory
	pkgDirs := make(map[string]string)
	// root directories of packages that we want to extract
	wantedRoots := make(map[string]bool)

	// recursively visit all packages in depth-first order;
	// on the way down, associate each package scope with its corresponding package,
	// and on the way up extract the package's scope
	packages.Visit(pkgs, func(pkg *packages.Package) bool {
		return true
	}, func(pkg *packages.Package) {
		if _, ok := pkgRoots[pkg.PkgPath]; !ok {
			mdir := util.GetModDir(pkg.PkgPath, modFlags...)
			pdir := util.GetPkgDir(pkg.PkgPath, modFlags...)
			// GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package
			// is not using modules. If this is the case, fall back to the package directory
			if mdir == "" {
				mdir = pdir
			}
			pkgRoots[pkg.PkgPath] = mdir
			pkgDirs[pkg.PkgPath] = pdir
		}

		tw, err := trap.NewWriter(pkg.PkgPath, pkg)
		if err != nil {
			log.Fatal(err)
		}
		defer tw.Close()

		scope := extractPackageScope(tw, pkg)
		tw.ForEachObject(extractObjectType)
		lbl := tw.Labeler.GlobalID(pkg.PkgPath + ";pkg")
		dbscheme.PackagesTable.Emit(tw, lbl, pkg.Name, pkg.PkgPath, scope)

		if len(pkg.Errors) != 0 {
			log.Printf("Warning: encountered errors extracting package `%s`:", pkg.PkgPath)
			for i, err := range pkg.Errors {
				log.Printf("  %s", err.Error())
				extractError(tw, err, lbl, i)
			}
		}
	})

	for _, pkg := range pkgs {
		if pkgRoots[pkg.PkgPath] == "" {
			log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)
		}
		wantedRoots[pkgRoots[pkg.PkgPath]] = true
	}

	// this sets the number of threads that the Go runtime will spawn; this is separate
	// from the number of goroutines that the program spawns, which are scheduled into
	// the system threads by the Go runtime scheduler
	threads := os.Getenv("LGTM_THREADS")
	if maxprocs, err := strconv.Atoi(threads); err == nil && maxprocs > 0 {
		log.Printf("Max threads set to %d", maxprocs)
		runtime.GOMAXPROCS(maxprocs)
	} else if threads != "" {
		log.Printf("Warning: LGTM_THREADS value %s is not valid, defaulting to using all available threads.", threads)
	}
	// if the value is empty or not set, use the Go default, which is the number of cores
	// available since Go 1.5, but is subject to change

	var maxgoroutines int
	if maxgoroutines, err = strconv.Atoi(util.Getenv(
		"CODEQL_EXTRACTOR_GO_MAX_GOROUTINES",
		"SEMMLE_MAX_GOROUTINES",
	)); err != nil {
		maxgoroutines = 32
	} else {
		log.Printf("Max goroutines set to %d", maxgoroutines)
	}

	var wg sync.WaitGroup
	// this semaphore is used to limit the number of files that are open at once;
	// this is to prevent the extractor from running into issues with caps on the
	// number of open files that can be held by one process
	fdSem := newSemaphore(100)
	// this semaphore is used to limit the number of goroutines spawned, so we
	// don't run into memory issues
	goroutineSem := newSemaphore(maxgoroutines)

	// extract AST information for all packages
	packages.Visit(pkgs, func(pkg *packages.Package) bool {
		return true
	}, func(pkg *packages.Package) {
		sep := regexp.QuoteMeta(string(filepath.Separator))
		relativeOrVendorRe := regexp.MustCompile(`(^\.\.` + sep + ")|((^|.*" + sep + ")vendor" + sep + ").*")

		for root, _ := range wantedRoots {
			relDir, err := filepath.Rel(root, pkgDirs[pkg.PkgPath])
			if err != nil || relativeOrVendorRe.MatchString(relDir) {
				// if the paths can't be made relative or the relative path starts with `".."` or contains a
				// directory called `"vendor"`, skip it
				continue
			}

			extractPackage(pkg, &wg, goroutineSem, fdSem)
			return
		}
	})

	wg.Wait()

	cwd, err := os.Getwd()
	if err != nil {
		log.Printf("Warning: unable to get working directory: %s", err.Error())
		log.Println("Skipping go.mod extraction")
	}
	rcwd, err := filepath.EvalSymlinks(cwd)
	if err == nil {
		cwd = rcwd
	}

	goModPaths := make([]string, 0, 10)

	filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
		if filepath.Base(path) == "go.mod" && info != nil && info.Mode().IsRegular() {
			if err != nil {
				log.Printf("Found go.mod with path %s, but encountered error %s", path, err.Error())
			}

			goModPaths = append(goModPaths, path)
		}<commit_msg>extractor: Refactor regexp compilation for the relative directory check<commit_after>package extractor

import (
	"fmt"
	"go/ast"
	"go/constant"
	"go/scanner"
	"go/token"
	"go/types"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/github/codeql-go/extractor/dbscheme"
	"github.com/github/codeql-go/extractor/srcarchive"
	"github.com/github/codeql-go/extractor/trap"
	"github.com/github/codeql-go/extractor/util"
	"golang.org/x/tools/go/packages"
)

// Extract extracts the packages specified by the given patterns
func Extract(patterns []string) error {
	return ExtractWithFlags(nil, patterns)
}

// ExtractWithFlags extracts the packages specified by the given patterns and build flags
func ExtractWithFlags(buildFlags []string, patterns []string) error {
	cfg := &packages.Config{
		Mode: packages.NeedName | packages.NeedFiles |
			packages.NeedCompiledGoFiles |
			packages.NeedImports | packages.NeedDeps |
			packages.NeedTypes | packages.NeedTypesSizes |
			packages.NeedTypesInfo | packages.NeedSyntax,
		BuildFlags: buildFlags,
	}
	pkgs, err := packages.Load(cfg, patterns...)

	modFlags := make([]string, 0, 1)
	for _, flag := range buildFlags {
		if strings.HasPrefix(flag, "-mod=") {
			modFlags = append(modFlags, flag)
		}
	}

	if err != nil {
		return err
	}

	if len(pkgs) == 0 {
		log.Printf("No packages found.")
	}

	extractUniverseScope()

	// a map of package path to package root directory (currently the module root or the source directory)
	pkgRoots := make(map[string]string)
	// a map of package path to source code directory
	pkgDirs := make(map[string]string)
	// root directories of packages that we want to extract
	wantedRoots := make(map[string]bool)

	// recursively visit all packages in depth-first order;
	// on the way down, associate each package scope with its corresponding package,
	// and on the way up extract the package's scope
	packages.Visit(pkgs, func(pkg *packages.Package) bool {
		return true
	}, func(pkg *packages.Package) {
		if _, ok := pkgRoots[pkg.PkgPath]; !ok {
			mdir := util.GetModDir(pkg.PkgPath, modFlags...)
			pdir := util.GetPkgDir(pkg.PkgPath, modFlags...)
			// GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package
			// is not using modules. If this is the case, fall back to the package directory
			if mdir == "" {
				mdir = pdir
			}
			pkgRoots[pkg.PkgPath] = mdir
			pkgDirs[pkg.PkgPath] = pdir
		}

		tw, err := trap.NewWriter(pkg.PkgPath, pkg)
		if err != nil {
			log.Fatal(err)
		}
		defer tw.Close()

		scope := extractPackageScope(tw, pkg)
		tw.ForEachObject(extractObjectType)
		lbl := tw.Labeler.GlobalID(pkg.PkgPath + ";pkg")
		dbscheme.PackagesTable.Emit(tw, lbl, pkg.Name, pkg.PkgPath, scope)

		if len(pkg.Errors) != 0 {
			log.Printf("Warning: encountered errors extracting package `%s`:", pkg.PkgPath)
			for i, err := range pkg.Errors {
				log.Printf("  %s", err.Error())
				extractError(tw, err, lbl, i)
			}
		}
	})

	for _, pkg := range pkgs {
		if pkgRoots[pkg.PkgPath] == "" {
			log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)
		}
		wantedRoots[pkgRoots[pkg.PkgPath]] = true
	}

	// this sets the number of threads that the Go runtime will spawn; this is separate
	// from the number of goroutines that the program spawns, which are scheduled into
	// the system threads by the Go runtime scheduler
	threads := os.Getenv("LGTM_THREADS")
	if maxprocs, err := strconv.Atoi(threads); err == nil && maxprocs > 0 {
		log.Printf("Max threads set to %d", maxprocs)
		runtime.GOMAXPROCS(maxprocs)
	} else if threads != "" {
		log.Printf("Warning: LGTM_THREADS value %s is not valid, defaulting to using all available threads.", threads)
	}
	// if the value is empty or not set, use the Go default, which is the number of cores
	// available since Go 1.5, but is subject to change

	var maxgoroutines int
	if maxgoroutines, err = strconv.Atoi(util.Getenv(
		"CODEQL_EXTRACTOR_GO_MAX_GOROUTINES",
		"SEMMLE_MAX_GOROUTINES",
	)); err != nil {
		maxgoroutines = 32
	} else {
		log.Printf("Max goroutines set to %d", maxgoroutines)
	}

	var wg sync.WaitGroup
	// this semaphore is used to limit the number of files that are open at once;
	// this is to prevent the extractor from running into issues with caps on the
	// number of open files that can be held by one process
	fdSem := newSemaphore(100)
	// this semaphore is used to limit the number of goroutines spawned, so we
	// don't run into memory issues
	goroutineSem := newSemaphore(maxgoroutines)

	sep := regexp.QuoteMeta(string(filepath.Separator))
	// if a path matches this regexp, we don't want to extract this package. Currently, it checks
	//   - that the path does not contain a `..` segment, and
	//   - the path does not contain a `vendor` directory.
	noExtractRe := regexp.MustCompile(`.*(^|` + sep + `)(\.\.|vendor)($|` + sep + `).*`)

	// extract AST information for all packages
	packages.Visit(pkgs, func(pkg *packages.Package) bool {
		return true
	}, func(pkg *packages.Package) {
		for root, _ := range wantedRoots {
			relDir, err := filepath.Rel(root, pkgDirs[pkg.PkgPath])
			if err != nil || noExtractRe.MatchString(relDir) {
				// if the path can't be made relative or matches the noExtract regexp skip it
				continue
			}

			extractPackage(pkg, &wg, goroutineSem, fdSem)
			return
		}
	})

	wg.Wait()

	cwd, err := os.Getwd()
	if err != nil {
		log.Printf("Warning: unable to get working directory: %s", err.Error())
		log.Println("Skipping go.mod extraction")
	}
	rcwd, err := filepath.EvalSymlinks(cwd)
	if err == nil {
		cwd = rcwd
	}

	goModPaths := make([]string, 0, 10)

	filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
		if filepath.Base(path) == "go.mod" && info != nil && info.Mode().IsRegular() {
			if err != nil {
				log.Printf("Found go.mod with path %s, but encountered error %s", path, err.Error())
			}

			goModPaths = append(goModPaths, path)
		}
{'fraction_non_alphanumeric': 0.12488389766106561, 'fraction_numerical': 0.0025331419403867264, 'mean_word_length': 3.714968152866242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7449386', 'n_tokens_mistral': 4359, 'n_tokens_neox': 4006, 'n_words': 1225}