<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/google/safehtml"
	"github.com/google/safehtml/uncheckedconversions"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/cookie"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/stdlib"
)

// UnitPage contains data needed to render the unit template.
type UnitPage struct {
	basePage
	// Unit is the unit for this page.
	Unit *internal.UnitMeta

	// Breadcrumb contains data used to render breadcrumb UI elements.
	Breadcrumb breadcrumb

	// Title is the title of the page.
	Title string

	// URLPath is the path suitable for links on the page.
	// See the unitURLPath for details.
	URLPath string

	// CanonicalURLPath is a permanent representation of the URL path for a
	// unit.
	// It uses the resolved module path and version.
	// For example, if the latest version of /my.module/pkg is version v1.5.2,
	// the canonical URL path for that unit would be /my.module@v1.5.2/pkg
	CanonicalURLPath string

	// The version string formatted for display.
	DisplayVersion string

	// LinkVersion is version string suitable for links used to compute
	// latest badges.
	LinkVersion string

	// LatestURL is a url pointing to the latest version of a unit.
	LatestURL string

	// LatestMinorClass is the CSS class that describes the current unit's minor
	// version in relationship to the latest version of the unit.
	LatestMinorClass string

	// Information about the latest major version of the module.
	LatestMajorVersion    string
	LatestMajorVersionURL string

	// PageType is the type of page (pkg, cmd, dir, std, or mod).
	PageType string

	// PageLabels are the labels that will be displayed
	// for a given page.
	PageLabels []string

	// CanShowDetails indicates whether details can be shown or must be
	// hidden due to issues like license restrictions.
	CanShowDetails bool

	// Settings contains settings for the selected tab.
	SelectedTab TabSettings

	// RedirectedFromPath is the path that redirected to the current page.
	// If non-empty, a "redirected from" banner will be displayed
	// (see content/static/html/helpers/_unit_header.tmpl).
	RedirectedFromPath string

	// Details contains data specific to the type of page being rendered.
	Details interface{}
}

// serveUnitPage serves a unit page for a path using the paths,
// modules, documentation, readmes, licenses, and package_imports tables.
func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *http.Request,
	ds internal.DataSource, info *urlPathInfo) (err error) {
	defer derrors.Wrap(&err, "serveUnitPage(ctx, w, r, ds, %v)", info)

	tab := r.FormValue("tab")
	if tab == "" {
		// Default to details tab when there is no tab param.
		tab = tabMain
	}
	// Redirect to clean URL path when tab param is invalid.
	if _, ok := unitTabLookup[tab]; !ok {
		http.Redirect(w, r, r.URL.Path, http.StatusFound)
		return nil
	}

	um, err := ds.GetUnitMeta(ctx, info.fullPath, info.modulePath, info.requestedVersion)
	if err != nil {
		if !errors.Is(err, derrors.NotFound) {
			return err
		}
		return s.servePathNotFoundPage(w, r, ds, info.fullPath, info.modulePath, info.requestedVersion)
	}

	// Use GOOS and GOARCH query parameters to create a build context, which
	// affects the documentation and synopsis. Omitting both results in an empty
	// build context, which will match the first (and preferred) build context.
	// It's also okay to provide just one (e.g. GOOS=windows), which will select
	// the first doc with that value, ignoring the other one.
	bc := internal.BuildContext{GOOS: r.FormValue("GOOS"), GOARCH: r.FormValue("GOARCH")}
	d, err := fetchDetailsForUnit(ctx, r, tab, ds, um, bc)
	if err != nil {
		return err
	}
	if s.serveStats && r.FormValue("m") == "json" {
		data, err := json.Marshal(d)
		if err != nil {
			return fmt.Errorf("json.Marshal: %v", err)
		}
		if _, err := w.Write(data); err != nil {
			return fmt.Errorf("w.Write: %v", err)
		}
		return nil
	}

	recordVersionTypeMetric(ctx, info.requestedVersion)
	if _, ok := internal.DefaultBranches[info.requestedVersion]; ok {
		// Since path@master is a moving target, we don't want it to be stale.
		// As a result, we enqueue every request of path@master to the frontend
		// task queue, which will initiate a fetch request depending on the
		// last time we tried to fetch this module version.
		//
		// Use a separate context here to prevent the context from being canceled
		// elsewhere before a task is enqueued.
		go func() {
			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
			defer cancel()
			log.Infof(ctx, "serveUnitPage: Scheduling %q@%q to be fetched", info.modulePath, info.requestedVersion)
			if _, err := s.queue.ScheduleFetch(ctx, info.modulePath, info.requestedVersion, "", false); err != nil {
				log.Errorf(ctx, "serveUnitPage(%q): %v", r.URL.Path, err)
			}
		}()
	}

	if !isValidTabForUnit(tab, um) {
		// Redirect to clean URL path when tab param is invalid for the unit
		// type.
		http.Redirect(w, r, r.URL.Path, http.StatusFound)
		return nil
	}

	// If we've already called GetUnitMeta for an unknown module path and the latest version, pass
	// it to GetLatestInfo to avoid a redundant call.
	var latestUnitMeta *internal.UnitMeta
	if info.modulePath == internal.UnknownModulePath && info.requestedVersion == internal.LatestVersion {
		latestUnitMeta = um
	}
	latestInfo := s.GetLatestInfo(ctx, um.Path, um.ModulePath, latestUnitMeta)<commit_msg>internal/frontend: fix unknown module path in ScheduleFetch<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/google/safehtml"
	"github.com/google/safehtml/uncheckedconversions"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/cookie"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/stdlib"
)

// UnitPage contains data needed to render the unit template.
type UnitPage struct {
	basePage
	// Unit is the unit for this page.
	Unit *internal.UnitMeta

	// Breadcrumb contains data used to render breadcrumb UI elements.
	Breadcrumb breadcrumb

	// Title is the title of the page.
	Title string

	// URLPath is the path suitable for links on the page.
	// See the unitURLPath for details.
	URLPath string

	// CanonicalURLPath is a permanent representation of the URL path for a
	// unit.
	// It uses the resolved module path and version.
	// For example, if the latest version of /my.module/pkg is version v1.5.2,
	// the canonical URL path for that unit would be /my.module@v1.5.2/pkg
	CanonicalURLPath string

	// The version string formatted for display.
	DisplayVersion string

	// LinkVersion is version string suitable for links used to compute
	// latest badges.
	LinkVersion string

	// LatestURL is a url pointing to the latest version of a unit.
	LatestURL string

	// LatestMinorClass is the CSS class that describes the current unit's minor
	// version in relationship to the latest version of the unit.
	LatestMinorClass string

	// Information about the latest major version of the module.
	LatestMajorVersion    string
	LatestMajorVersionURL string

	// PageType is the type of page (pkg, cmd, dir, std, or mod).
	PageType string

	// PageLabels are the labels that will be displayed
	// for a given page.
	PageLabels []string

	// CanShowDetails indicates whether details can be shown or must be
	// hidden due to issues like license restrictions.
	CanShowDetails bool

	// Settings contains settings for the selected tab.
	SelectedTab TabSettings

	// RedirectedFromPath is the path that redirected to the current page.
	// If non-empty, a "redirected from" banner will be displayed
	// (see content/static/html/helpers/_unit_header.tmpl).
	RedirectedFromPath string

	// Details contains data specific to the type of page being rendered.
	Details interface{}
}

// serveUnitPage serves a unit page for a path using the paths,
// modules, documentation, readmes, licenses, and package_imports tables.
func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *http.Request,
	ds internal.DataSource, info *urlPathInfo) (err error) {
	defer derrors.Wrap(&err, "serveUnitPage(ctx, w, r, ds, %v)", info)

	tab := r.FormValue("tab")
	if tab == "" {
		// Default to details tab when there is no tab param.
		tab = tabMain
	}
	// Redirect to clean URL path when tab param is invalid.
	if _, ok := unitTabLookup[tab]; !ok {
		http.Redirect(w, r, r.URL.Path, http.StatusFound)
		return nil
	}

	um, err := ds.GetUnitMeta(ctx, info.fullPath, info.modulePath, info.requestedVersion)
	if err != nil {
		if !errors.Is(err, derrors.NotFound) {
			return err
		}
		return s.servePathNotFoundPage(w, r, ds, info.fullPath, info.modulePath, info.requestedVersion)
	}

	// Use GOOS and GOARCH query parameters to create a build context, which
	// affects the documentation and synopsis. Omitting both results in an empty
	// build context, which will match the first (and preferred) build context.
	// It's also okay to provide just one (e.g. GOOS=windows), which will select
	// the first doc with that value, ignoring the other one.
	bc := internal.BuildContext{GOOS: r.FormValue("GOOS"), GOARCH: r.FormValue("GOARCH")}
	d, err := fetchDetailsForUnit(ctx, r, tab, ds, um, bc)
	if err != nil {
		return err
	}
	if s.serveStats && r.FormValue("m") == "json" {
		data, err := json.Marshal(d)
		if err != nil {
			return fmt.Errorf("json.Marshal: %v", err)
		}
		if _, err := w.Write(data); err != nil {
			return fmt.Errorf("w.Write: %v", err)
		}
		return nil
	}

	recordVersionTypeMetric(ctx, info.requestedVersion)
	if _, ok := internal.DefaultBranches[info.requestedVersion]; ok {
		// Since path@master is a moving target, we don't want it to be stale.
		// As a result, we enqueue every request of path@master to the frontend
		// task queue, which will initiate a fetch request depending on the
		// last time we tried to fetch this module version.
		//
		// Use a separate context here to prevent the context from being canceled
		// elsewhere before a task is enqueued.
		go func() {
			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
			defer cancel()
			log.Infof(ctx, "serveUnitPage: Scheduling %q@%q to be fetched", um.ModulePath, info.requestedVersion)
			if _, err := s.queue.ScheduleFetch(ctx, um.ModulePath, info.requestedVersion, "", false); err != nil {
				log.Errorf(ctx, "serveUnitPage(%q): scheduling fetch for %q@%q: %v",
					r.URL.Path, um.ModulePath, info.requestedVersion, err)
			}
		}()
	}

	if !isValidTabForUnit(tab, um) {
		// Redirect to clean URL path when tab param is invalid for the unit
		// type.
		http.Redirect(w, r, r.URL.Path, http.StatusFound)
		return nil
	}

	// If we've already called GetUnitMeta for an unknown module path and the latest version, pass
	// it to GetLatestInfo to avoid a redundant call.
	var latestUnitMeta *internal.UnitMeta
	if info.modulePath == internal.UnknownModulePath && info.requestedVersion == internal.LatestVersion {
		latestUnitMeta = um
	}
	latestInfo := s.GetLatestInfo(ctx, um.Path, um.ModulePath, latestUnitMeta)
{'fraction_non_alphanumeric': 0.10023249806251615, 'fraction_numerical': 0.001894428657538965, 'mean_word_length': 3.6698833936469644, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '890838', 'n_tokens_mistral': 3751, 'n_tokens_neox': 3536, 'n_words': 1349}