<commit_before>package extractor

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"go/ast"
	"go/constant"
	"go/scanner"
	"go/token"
	"go/types"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/github/codeql-go/extractor/dbscheme"
	"github.com/github/codeql-go/extractor/srcarchive"
	"github.com/github/codeql-go/extractor/trap"
	"github.com/github/codeql-go/extractor/util"
	"golang.org/x/tools/go/packages"
)

var MaxGoRoutines int

func init() {
	// this sets the number of threads that the Go runtime will spawn; this is separate
	// from the number of goroutines that the program spawns, which are scheduled into
	// the system threads by the Go runtime scheduler
	threads := os.Getenv("LGTM_THREADS")
	if maxprocs, err := strconv.Atoi(threads); err == nil && maxprocs > 0 {
		log.Printf("Max threads set to %d", maxprocs)
		runtime.GOMAXPROCS(maxprocs)
	} else if threads != "" {
		log.Printf("Warning: LGTM_THREADS value %s is not valid, defaulting to using all available threads.", threads)
	}
	// if the value is empty or not set, use the Go default, which is the number of cores
	// available since Go 1.5, but is subject to change

	var err error
	if MaxGoRoutines, err = strconv.Atoi(util.Getenv(
		"CODEQL_EXTRACTOR_GO_MAX_GOROUTINES",
		"SEMMLE_MAX_GOROUTINES",
	)); err != nil {
		MaxGoRoutines = 32
	} else {
		log.Printf("Max goroutines set to %d", MaxGoRoutines)
	}
}

// Extract extracts the packages specified by the given patterns
func Extract(patterns []string) error {
	return ExtractWithFlags(nil, patterns)
}

// ExtractWithFlags extracts the packages specified by the given patterns and build flags
func ExtractWithFlags(buildFlags []string, patterns []string) error {
	startTime := time.Now()

	extraction := NewExtraction(buildFlags, patterns)
	defer extraction.StatWriter.Close()

	modEnabled := os.Getenv("GO111MODULE") != "off"
	if !modEnabled {
		log.Println("Go module mode disabled.")
	}

	modFlags := make([]string, 0, 1)
	for _, flag := range buildFlags {
		if strings.HasPrefix(flag, "-mod=") {
			modFlags = append(modFlags, flag)
		}
	}

	log.Println("Running packages.Load.")
	cfg := &packages.Config{
		Mode: packages.NeedName | packages.NeedFiles |
			packages.NeedCompiledGoFiles |
			packages.NeedImports | packages.NeedDeps |
			packages.NeedTypes | packages.NeedTypesSizes |
			packages.NeedTypesInfo | packages.NeedSyntax,
		BuildFlags: buildFlags,
	}
	pkgs, err := packages.Load(cfg, patterns...)
	if err != nil {
		return err
	}
	log.Println("Done running packages.Load.")

	if len(pkgs) == 0 {
		log.Println("No packages found.")
	}

	log.Println("Extracting universe scope.")
	extractUniverseScope()
	log.Println("Done extracting universe scope.")

	// a map of package path to package root directory (currently the module root or the source directory)
	pkgRoots := make(map[string]string)
	// a map of package path to source code directory
	pkgDirs := make(map[string]string)
	// root directories of packages that we want to extract
	wantedRoots := make(map[string]bool)

	// recursively visit all packages in depth-first order;
	// on the way down, associate each package scope with its corresponding package,
	// and on the way up extract the package's scope
	packages.Visit(pkgs, func(pkg *packages.Package) bool {
		return true
	}, func(pkg *packages.Package) {
		log.Printf("Processing package %s.", pkg.PkgPath)

		if _, ok := pkgRoots[pkg.PkgPath]; !ok {
			mdir := util.GetModDir(pkg.PkgPath, modFlags...)
			pdir := util.GetPkgDir(pkg.PkgPath, modFlags...)
			// GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package
			// is not using modules. If this is the case, fall back to the package directory
			if mdir == "" {
				mdir = pdir
			}
			pkgRoots[pkg.PkgPath] = mdir
			pkgDirs[pkg.PkgPath] = pdir
		}

		log.Printf("Extracting types for package %s.", pkg.PkgPath)

		tw, err := trap.NewWriter(pkg.PkgPath, pkg)
		if err != nil {
			log.Fatal(err)
		}
		defer tw.Close()

		scope := extractPackageScope(tw, pkg)
		extractObjectTypes(tw)
		lbl := tw.Labeler.GlobalID(util.EscapeTrapSpecialChars(pkg.PkgPath) + ";pkg")
		dbscheme.PackagesTable.Emit(tw, lbl, pkg.Name, pkg.PkgPath, scope)<commit_msg>Update comment above first extraction of packages<commit_after>package extractor

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"go/ast"
	"go/constant"
	"go/scanner"
	"go/token"
	"go/types"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/github/codeql-go/extractor/dbscheme"
	"github.com/github/codeql-go/extractor/srcarchive"
	"github.com/github/codeql-go/extractor/trap"
	"github.com/github/codeql-go/extractor/util"
	"golang.org/x/tools/go/packages"
)

var MaxGoRoutines int

func init() {
	// this sets the number of threads that the Go runtime will spawn; this is separate
	// from the number of goroutines that the program spawns, which are scheduled into
	// the system threads by the Go runtime scheduler
	threads := os.Getenv("LGTM_THREADS")
	if maxprocs, err := strconv.Atoi(threads); err == nil && maxprocs > 0 {
		log.Printf("Max threads set to %d", maxprocs)
		runtime.GOMAXPROCS(maxprocs)
	} else if threads != "" {
		log.Printf("Warning: LGTM_THREADS value %s is not valid, defaulting to using all available threads.", threads)
	}
	// if the value is empty or not set, use the Go default, which is the number of cores
	// available since Go 1.5, but is subject to change

	var err error
	if MaxGoRoutines, err = strconv.Atoi(util.Getenv(
		"CODEQL_EXTRACTOR_GO_MAX_GOROUTINES",
		"SEMMLE_MAX_GOROUTINES",
	)); err != nil {
		MaxGoRoutines = 32
	} else {
		log.Printf("Max goroutines set to %d", MaxGoRoutines)
	}
}

// Extract extracts the packages specified by the given patterns
func Extract(patterns []string) error {
	return ExtractWithFlags(nil, patterns)
}

// ExtractWithFlags extracts the packages specified by the given patterns and build flags
func ExtractWithFlags(buildFlags []string, patterns []string) error {
	startTime := time.Now()

	extraction := NewExtraction(buildFlags, patterns)
	defer extraction.StatWriter.Close()

	modEnabled := os.Getenv("GO111MODULE") != "off"
	if !modEnabled {
		log.Println("Go module mode disabled.")
	}

	modFlags := make([]string, 0, 1)
	for _, flag := range buildFlags {
		if strings.HasPrefix(flag, "-mod=") {
			modFlags = append(modFlags, flag)
		}
	}

	log.Println("Running packages.Load.")
	cfg := &packages.Config{
		Mode: packages.NeedName | packages.NeedFiles |
			packages.NeedCompiledGoFiles |
			packages.NeedImports | packages.NeedDeps |
			packages.NeedTypes | packages.NeedTypesSizes |
			packages.NeedTypesInfo | packages.NeedSyntax,
		BuildFlags: buildFlags,
	}
	pkgs, err := packages.Load(cfg, patterns...)
	if err != nil {
		return err
	}
	log.Println("Done running packages.Load.")

	if len(pkgs) == 0 {
		log.Println("No packages found.")
	}

	log.Println("Extracting universe scope.")
	extractUniverseScope()
	log.Println("Done extracting universe scope.")

	// a map of package path to package root directory (currently the module root or the source directory)
	pkgRoots := make(map[string]string)
	// a map of package path to source code directory
	pkgDirs := make(map[string]string)
	// root directories of packages that we want to extract
	wantedRoots := make(map[string]bool)

	// Do a post-order traversal and extract the package scope of each package
	packages.Visit(pkgs, func(pkg *packages.Package) bool {
		return true
	}, func(pkg *packages.Package) {
		log.Printf("Processing package %s.", pkg.PkgPath)

		if _, ok := pkgRoots[pkg.PkgPath]; !ok {
			mdir := util.GetModDir(pkg.PkgPath, modFlags...)
			pdir := util.GetPkgDir(pkg.PkgPath, modFlags...)
			// GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package
			// is not using modules. If this is the case, fall back to the package directory
			if mdir == "" {
				mdir = pdir
			}
			pkgRoots[pkg.PkgPath] = mdir
			pkgDirs[pkg.PkgPath] = pdir
		}

		log.Printf("Extracting types for package %s.", pkg.PkgPath)

		tw, err := trap.NewWriter(pkg.PkgPath, pkg)
		if err != nil {
			log.Fatal(err)
		}
		defer tw.Close()

		scope := extractPackageScope(tw, pkg)
		extractObjectTypes(tw)
		lbl := tw.Labeler.GlobalID(util.EscapeTrapSpecialChars(pkg.PkgPath) + ";pkg")
		dbscheme.PackagesTable.Emit(tw, lbl, pkg.Name, pkg.PkgPath, scope)
{'fraction_non_alphanumeric': 0.12230470130788264, 'fraction_numerical': 0.002827854365500177, 'mean_word_length': 4.043374925727867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3476959', 'n_tokens_mistral': 3087, 'n_tokens_neox': 2867, 'n_words': 817}