<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

package main

import (
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/net/context/ctxhttp"
)

type pkggodevEvent struct {
	Host    string
	Path    string
	Status  int
	URL     string
	Latency time.Duration
	Error   error
}

func teeRequestToPkgGoDev(godocReq *http.Request, latency time.Duration, isRobot bool, status int) (gddoEvent *gddoEvent, pkgEvent *pkggodevEvent) {
	gddoEvent = newGDDOEvent(godocReq, latency, isRobot, status)
	u := pkgGoDevURL(godocReq.URL)

	// Strip the utm_source from the URL.
	vals := u.Query()
	vals.Del("utm_source")
	u.RawQuery = vals.Encode()

	pkgEvent = &pkggodevEvent{
		Host: u.Host,
		Path: u.Path,
		URL:  u.String(),
	}
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		pkgEvent.Status = http.StatusInternalServerError
		pkgEvent.Error = err
		return gddoEvent, pkgEvent
	}
	start := time.Now()
	xfwd := req.Header.Get("X-Forwarded-for")
	req.Header.Set("X-Godoc-Forwarded-for", xfwd)
	resp, err := ctxhttp.Do(godocReq.Context(), http.DefaultClient, req)
	if err != nil {
		// Use StatusBadGateway to indicate the upstream error.
		pkgEvent.Status = http.StatusBadGateway
		pkgEvent.Error = err
		return gddoEvent, pkgEvent
	}

	pkgEvent.Status = resp.StatusCode
	pkgEvent.Latency = time.Since(start)
	return gddoEvent, pkgEvent

}

// doNotTeeURLsToPkgGoDev are paths that should not be teed to pkg.go.dev.<commit_msg>gddo-server: fetch path on pkg.go.dev 404<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/net/context/ctxhttp"
)

type pkggodevEvent struct {
	Host    string
	Path    string
	Status  int
	URL     string
	Latency time.Duration
	Error   string
	// If a request 404s, make a request to fetch it and store the response.
	FetchStatus   int
	FetchResponse string
}

func teeRequestToPkgGoDev(godocReq *http.Request, latency time.Duration, isRobot bool, status int) (gddoEvent *gddoEvent, pkgEvent *pkggodevEvent) {
	gddoEvent = newGDDOEvent(godocReq, latency, isRobot, status)
	u := pkgGoDevURL(godocReq.URL)

	// Strip the utm_source from the URL.
	vals := u.Query()
	vals.Del("utm_source")
	u.RawQuery = vals.Encode()

	pkgEvent = &pkggodevEvent{
		Host: u.Host,
		Path: u.Path,
		URL:  u.String(),
	}
	start := time.Now()
	status, errResp := makeRequest(godocReq.Context(), u.String())
	pkgEvent.Status = status
	pkgEvent.Latency = time.Since(start)
	// The response will always be an error here if not empty.
	pkgEvent.Error = errResp

	if pkgEvent.Status == http.StatusNotFound && gddoEvent.Status == http.StatusOK {
		// If the request was successful on godoc.org but returned a 404 on
		// pkg.go.dev make a fetch request.
		status, body := makeRequest(godocReq.Context(), "/fetch"+u.String())
		pkgEvent.FetchStatus = status
		pkgEvent.FetchResponse = body
	}
	return gddoEvent, pkgEvent
}

func makeRequest(ctx context.Context, url string) (int, string) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return http.StatusInternalServerError, fmt.Sprintf("http.NewRequest: %v", err)
	}
	xfwd := req.Header.Get("X-Forwarded-for")
	req.Header.Set("X-Godoc-Forwarded-for", xfwd)
	resp, err := ctxhttp.Do(ctx, http.DefaultClient, req)
	if err != nil {
		// Use StatusBadGateway to indicate the upstream error.
		return http.StatusBadGateway, err.Error()
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return resp.StatusCode, fmt.Sprintf("can't read body: %v", err)
	}
	return resp.StatusCode, string(body)
}

// doNotTeeURLsToPkgGoDev are paths that should not be teed to pkg.go.dev.
{'fraction_non_alphanumeric': 0.12205203014831023, 'fraction_numerical': 0.004133236080719669, 'mean_word_length': 3.857142857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '732923', 'n_tokens_mistral': 1537, 'n_tokens_neox': 1466, 'n_words': 399}