<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/mod/modfile"
	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/lsp/protocol"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/span"
	"golang.org/x/tools/internal/telemetry/log"
	"golang.org/x/tools/internal/telemetry/trace"
	errors "golang.org/x/xerrors"
)

const ModTidyError = "go mod tidy"
const SyntaxError = "syntax"

type parseModKey struct {
	snapshot source.Snapshot
	cfg      string
}

type modTidyHandle struct {
	handle *memoize.Handle
	file   source.FileHandle
	cfg    *packages.Config
}

type modTidyData struct {
	memoize.NoCopy

	// origfh is the file handle for the original go.mod file.
	origfh source.FileHandle

	// origParsedFile contains the parsed contents that are used to diff with
	// the ideal contents.
	origParsedFile *modfile.File

	// origMapper is the column mapper for the original go.mod file.
	origMapper *protocol.ColumnMapper

	// idealParsedFile contains the parsed contents for the go.mod file
	// after it has been "tidied".
	idealParsedFile *modfile.File

	// unusedDeps is the map containing the dependencies that are left after
	// removing the ones that are identical in the original and ideal go.mods.
	unusedDeps map[string]*modfile.Require

	// missingDeps is the map containing the dependencies that are left after
	// removing the ones that are identical in the original and ideal go.mods.
	missingDeps map[string]*modfile.Require

	// parseErrors are the errors that arise when we diff between a user's go.mod
	// and the "tidied" go.mod.
	parseErrors []source.Error

	// err is any error that occurs while we are calculating the parseErrors.
	err error
}

func (mth *modTidyHandle) String() string {
	return mth.File().Identity().URI.Filename()
}

func (mth *modTidyHandle) File() source.FileHandle {
	return mth.file
}

func (mth *modTidyHandle) Tidy(ctx context.Context) (*modfile.File, *protocol.ColumnMapper, map[string]*modfile.Require, []source.Error, error) {
	v := mth.handle.Get(ctx)
	if v == nil {
		return nil, nil, nil, nil, errors.Errorf("no parsed file for %s", mth.File().Identity().URI)
	}
	data := v.(*modTidyData)
	return data.origParsedFile, data.origMapper, data.missingDeps, data.parseErrors, data.err
}

func (s *snapshot) ModTidyHandle(ctx context.Context, realfh source.FileHandle) source.ModTidyHandle {
	realURI, tempURI := s.view.ModFiles()
	cfg := s.View().Config(ctx)
	options := s.View().Options()
	folder := s.View().Folder().Filename()

	key := parseModKey{
		snapshot: s,
		cfg:      hashConfig(cfg),
	}
	h := s.view.session.cache.store.Bind(key, func(ctx context.Context) interface{} {
		data := &modTidyData{}

		// Check the case when the tempModfile flag is turned off.
		if realURI == "" || tempURI == "" {
			return data
		}

		ctx, done := trace.StartSpan(ctx, "cache.ModTidyHandle", telemetry.File.Of(realURI))
		defer done()

		// Copy the real go.mod file content into the temp go.mod file.
		realContents, _, err := realfh.Read(ctx)
		if err != nil {
			data.err = err
			return data
		}
		if err := ioutil.WriteFile(tempURI.Filename(), realContents, os.ModePerm); err != nil {
			data.err = err
			return data
		}

		// We want to run "go mod tidy" to be able to diff between the real and the temp files.
		args := append([]string{"mod", "tidy"}, cfg.BuildFlags...)
		if _, err := source.InvokeGo(ctx, folder, cfg.Env, args...); err != nil {<commit_msg>internal/lsp/cache: use snapshot ID and view folder in ModTidyHandle key<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/mod/modfile"
	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/lsp/protocol"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/span"
	"golang.org/x/tools/internal/telemetry/log"
	"golang.org/x/tools/internal/telemetry/trace"
	errors "golang.org/x/xerrors"
)

const ModTidyError = "go mod tidy"
const SyntaxError = "syntax"

type parseModKey struct {
	view       string
	snapshotID uint64
	cfg        string
}

type modTidyHandle struct {
	handle *memoize.Handle
	file   source.FileHandle
	cfg    *packages.Config
}

type modTidyData struct {
	memoize.NoCopy

	// origfh is the file handle for the original go.mod file.
	origfh source.FileHandle

	// origParsedFile contains the parsed contents that are used to diff with
	// the ideal contents.
	origParsedFile *modfile.File

	// origMapper is the column mapper for the original go.mod file.
	origMapper *protocol.ColumnMapper

	// idealParsedFile contains the parsed contents for the go.mod file
	// after it has been "tidied".
	idealParsedFile *modfile.File

	// unusedDeps is the map containing the dependencies that are left after
	// removing the ones that are identical in the original and ideal go.mods.
	unusedDeps map[string]*modfile.Require

	// missingDeps is the map containing the dependencies that are left after
	// removing the ones that are identical in the original and ideal go.mods.
	missingDeps map[string]*modfile.Require

	// parseErrors are the errors that arise when we diff between a user's go.mod
	// and the "tidied" go.mod.
	parseErrors []source.Error

	// err is any error that occurs while we are calculating the parseErrors.
	err error
}

func (mth *modTidyHandle) String() string {
	return mth.File().Identity().URI.Filename()
}

func (mth *modTidyHandle) File() source.FileHandle {
	return mth.file
}

func (mth *modTidyHandle) Tidy(ctx context.Context) (*modfile.File, *protocol.ColumnMapper, map[string]*modfile.Require, []source.Error, error) {
	v := mth.handle.Get(ctx)
	if v == nil {
		return nil, nil, nil, nil, errors.Errorf("no parsed file for %s", mth.File().Identity().URI)
	}
	data := v.(*modTidyData)
	return data.origParsedFile, data.origMapper, data.missingDeps, data.parseErrors, data.err
}

func (s *snapshot) ModTidyHandle(ctx context.Context, realfh source.FileHandle) source.ModTidyHandle {
	realURI, tempURI := s.view.ModFiles()
	cfg := s.View().Config(ctx)
	options := s.View().Options()
	folder := s.View().Folder().Filename()

	key := parseModKey{
		view:       folder,
		snapshotID: s.ID(),
		cfg:        hashConfig(cfg),
	}
	h := s.view.session.cache.store.Bind(key, func(ctx context.Context) interface{} {
		data := &modTidyData{}

		// Check the case when the tempModfile flag is turned off.
		if realURI == "" || tempURI == "" {
			return data
		}

		ctx, done := trace.StartSpan(ctx, "cache.ModTidyHandle", telemetry.File.Of(realURI))
		defer done()

		// Copy the real go.mod file content into the temp go.mod file.
		realContents, _, err := realfh.Read(ctx)
		if err != nil {
			data.err = err
			return data
		}
		if err := ioutil.WriteFile(tempURI.Filename(), realContents, os.ModePerm); err != nil {
			data.err = err
			return data
		}

		// We want to run "go mod tidy" to be able to diff between the real and the temp files.
		args := append([]string{"mod", "tidy"}, cfg.BuildFlags...)
		if _, err := source.InvokeGo(ctx, folder, cfg.Env, args...); err != nil {
{'fraction_non_alphanumeric': 0.12207621550591327, 'fraction_numerical': 0.001314060446780552, 'mean_word_length': 3.7361543248288736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6176441', 'n_tokens_mistral': 2658, 'n_tokens_neox': 2511, 'n_words': 771}