<commit_before>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	// alternate TODO: make this an iota-based enum?
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// TODO: create an enum for use with "nometa" for readability's sake

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, rt)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := f.fetch(src)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	errBuilder := strings.Builder{}
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil {
			errBuilder.WriteString(fmt.Sprintf("%s => %s: %v\n", r.url, r.file, r.err))
		}
	}
	if errBuilder.Len() > 0 {
		return nil, errors.New(errBuilder.String())
	}

	return imap, nil
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err<commit_msg>Remove strings.Builder usage from fetch.go.<commit_after>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	// alternate TODO: make this an iota-based enum?
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// TODO: create an enum for use with "nometa" for readability's sake

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, rt)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := f.fetch(src)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	var errStr string
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil {
			errStr += fmt.Sprintf("%s => %s: %v\n", r.url, r.file, r.err)
		}
	}
	if len(errStr) > 0 {
		return nil, errors.New(errStr)
	}

	return imap, nil
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
{'fraction_non_alphanumeric': 0.1277296686746988, 'fraction_numerical': 0.005647590361445783, 'mean_word_length': 3.118217054263566, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2600310', 'n_tokens_mistral': 3947, 'n_tokens_neox': 3722, 'n_words': 1209}