<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package lib

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/loadimpact/k6/lib/fsext"
	"github.com/loadimpact/k6/loader"
	"github.com/spf13/afero"
)

//nolint: gochecknoglobals, lll
var (
	volumeRE  = regexp.MustCompile(`^[/\\]?([a-zA-Z]):(.*)`)
	sharedRE  = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js
	homeDirRE = regexp.MustCompile(`(?i)^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`)
)

// NormalizeAndAnonymizePath Normalizes (to use a / path separator) and anonymizes a file path, by scrubbing usernames from home directories.
func NormalizeAndAnonymizePath(path string) string {
	path = filepath.Clean(path)

	p := volumeRE.ReplaceAllString(path, `/$1$2`)
	p = sharedRE.ReplaceAllString(p, `/nobody`)
	p = strings.Replace(p, "\\", "/", -1)
	return homeDirRE.ReplaceAllString(p, `$1/$2/nobody`)
}

func newNormalizedFs(fs afero.Fs) afero.Fs {
	return fsext.NewChangePathFs(fs, fsext.ChangePathFunc(func(name string) (string, error) {
		return NormalizeAndAnonymizePath(name), nil
	}))
}

// An Archive is a rollup of all resources and options needed to reproduce a test identically elsewhere.
type Archive struct {
	// The runner to use, eg. "js".
	Type string `json:"type"`

	// Options to use.
	Options Options `json:"options"`

	// TODO: rewrite the encoding, decoding of json to use another type with only the fields it
	// needs in order to remove Filename and Pwd from this
	// Filename and contents of the main file being executed.
	Filename    string   `json:"filename"` // only for json
	FilenameURL *url.URL `json:"-"`
	Data        []byte   `json:"-"`

	// Working directory for resolving relative paths.
	Pwd    string   `json:"pwd"` // only for json
	PwdURL *url.URL `json:"-"`

	Filesystems map[string]afero.Fs `json:"-"`

	// Environment variables
	Env map[string]string `json:"env"`

	K6Version string `json:"k6version"`
	Goos      string `json:"goos"`
}

func (arc *Archive) getFs(name string) afero.Fs {
	fs, ok := arc.Filesystems[name]
	if !ok {
		fs = afero.NewMemMapFs()
		if name == "file" {
			fs = newNormalizedFs(fs)
		}
		arc.Filesystems[name] = fs
	}

	return fs
}

// cleanUpWrongMetadataJSON fixes issues with the metadata.json contents before
// they are unmarshalled in the Archive struct.
//
// Currently, the only fix this function performs is the discarding of the
// derived `execution` config value in the consolidated options that was wrongly
// saved by k6 in the archive metadata.json files until commit
// 83193f8a96e06a190325b838b2cc451119d6b836. This basically means k6 v0.24.0 and
// surrounding master commits. We filter these out by the value of the k6version
// property, saved in the metadata.json since the previous to the above commit.
func cleanUpWrongMetadataJSON(data []byte) ([]byte, error) {
	var tmpArc map[string]interface{}
	if err := json.Unmarshal(data, &tmpArc); err != nil {
		return nil, err
	}

	k6Version := ""
	if k6RawVersion, ok := tmpArc["k6version"]; ok {
		if k6Version, ok = k6RawVersion.(string); !ok {
			return nil, fmt.Errorf("k6version is present in the archive metadata, but it's not a string")
		}
	}

	// TODO: semantically parse the k6version and compare it with the current
	// one, log a warning if the current k6 version in lib/consts is lower than
	// the k6 version that generated the archive.

	if k6Version != "" && k6Version != "0.24.0" {
		return data, nil
	}

	if rawOptions, ok := tmpArc["options"]; !ok {
		return nil, fmt.Errorf("missing options key in the archive metadata.json")
	} else if options, ok := rawOptions.(map[string]interface{}); !ok {
		return nil, fmt.Errorf("wrong options type in metadata.json")
	} else if _, hasExecution := options["execution"]; !hasExecution {
		return data, nil // no need to fix anything
	} else {
		delete(options, "execution")
		tmpArc["options"] = options
	}

	return json.Marshal(tmpArc)
}

func (arc *Archive) loadMetadataJSON(data []byte) error {
	data, err := cleanUpWrongMetadataJSON(data)
	if err != nil {
		return err
	}

	if err = json.Unmarshal(data, &arc); err != nil {
		return err
	}
	// Path separator normalization for older archives (<=0.20.0)
	if arc.K6Version == "" {
		arc.Filename = NormalizeAndAnonymizePath(arc.Filename)
		arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd)
	}
	arc.PwdURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd)
	if err != nil {
		return err
	}
	arc.FilenameURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Filename)
	if err != nil {
		return err
	}<commit_msg>Export the archive CleanUpWrongMetadataJSON helper function<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package lib

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/loadimpact/k6/lib/fsext"
	"github.com/loadimpact/k6/loader"
	"github.com/spf13/afero"
)

//nolint: gochecknoglobals, lll
var (
	volumeRE  = regexp.MustCompile(`^[/\\]?([a-zA-Z]):(.*)`)
	sharedRE  = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js
	homeDirRE = regexp.MustCompile(`(?i)^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`)
)

// NormalizeAndAnonymizePath Normalizes (to use a / path separator) and anonymizes a file path, by scrubbing usernames from home directories.
func NormalizeAndAnonymizePath(path string) string {
	path = filepath.Clean(path)

	p := volumeRE.ReplaceAllString(path, `/$1$2`)
	p = sharedRE.ReplaceAllString(p, `/nobody`)
	p = strings.Replace(p, "\\", "/", -1)
	return homeDirRE.ReplaceAllString(p, `$1/$2/nobody`)
}

func newNormalizedFs(fs afero.Fs) afero.Fs {
	return fsext.NewChangePathFs(fs, fsext.ChangePathFunc(func(name string) (string, error) {
		return NormalizeAndAnonymizePath(name), nil
	}))
}

// An Archive is a rollup of all resources and options needed to reproduce a test identically elsewhere.
type Archive struct {
	// The runner to use, eg. "js".
	Type string `json:"type"`

	// Options to use.
	Options Options `json:"options"`

	// TODO: rewrite the encoding, decoding of json to use another type with only the fields it
	// needs in order to remove Filename and Pwd from this
	// Filename and contents of the main file being executed.
	Filename    string   `json:"filename"` // only for json
	FilenameURL *url.URL `json:"-"`
	Data        []byte   `json:"-"`

	// Working directory for resolving relative paths.
	Pwd    string   `json:"pwd"` // only for json
	PwdURL *url.URL `json:"-"`

	Filesystems map[string]afero.Fs `json:"-"`

	// Environment variables
	Env map[string]string `json:"env"`

	K6Version string `json:"k6version"`
	Goos      string `json:"goos"`
}

func (arc *Archive) getFs(name string) afero.Fs {
	fs, ok := arc.Filesystems[name]
	if !ok {
		fs = afero.NewMemMapFs()
		if name == "file" {
			fs = newNormalizedFs(fs)
		}
		arc.Filesystems[name] = fs
	}

	return fs
}

// CleanUpWrongMetadataJSON fixes issues with the metadata.json contents before
// they are unmarshalled in the Archive struct.
//
// Currently, the only fix this function performs is the discarding of the
// derived `execution` config value in the consolidated options that was wrongly
// saved by k6 in the archive metadata.json files until commit
// 83193f8a96e06a190325b838b2cc451119d6b836. This basically means k6 v0.24.0 and
// surrounding master commits. We filter these out by the value of the k6version
// property, saved in the metadata.json since the previous to the above commit.
func CleanUpWrongMetadataJSON(data []byte) ([]byte, error) {
	var tmpArc map[string]interface{}
	if err := json.Unmarshal(data, &tmpArc); err != nil {
		return nil, err
	}

	k6Version := ""
	if k6RawVersion, ok := tmpArc["k6version"]; ok {
		if k6Version, ok = k6RawVersion.(string); !ok {
			return nil, fmt.Errorf("k6version is present in the archive metadata, but it's not a string")
		}
	}

	// TODO: semantically parse the k6version and compare it with the current
	// one, log a warning if the current k6 version in lib/consts is lower than
	// the k6 version that generated the archive.

	if k6Version != "" && k6Version != "0.24.0" {
		return data, nil
	}

	if rawOptions, ok := tmpArc["options"]; !ok {
		return nil, fmt.Errorf("missing options key in the archive metadata.json")
	} else if options, ok := rawOptions.(map[string]interface{}); !ok {
		return nil, fmt.Errorf("wrong options type in metadata.json")
	} else if _, hasExecution := options["execution"]; !hasExecution {
		return data, nil // no need to fix anything
	} else {
		delete(options, "execution")
		tmpArc["options"] = options
	}

	return json.Marshal(tmpArc)
}

func (arc *Archive) loadMetadataJSON(data []byte) error {
	data, err := CleanUpWrongMetadataJSON(data)
	if err != nil {
		return err
	}

	if err = json.Unmarshal(data, &arc); err != nil {
		return err
	}
	// Path separator normalization for older archives (<=0.20.0)
	if arc.K6Version == "" {
		arc.Filename = NormalizeAndAnonymizePath(arc.Filename)
		arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd)
	}
	arc.PwdURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd)
	if err != nil {
		return err
	}
	arc.FilenameURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Filename)
	if err != nil {
		return err
	}
{'fraction_non_alphanumeric': 0.1268399055060876, 'fraction_numerical': 0.013628929674722878, 'mean_word_length': 3.8746678476527903, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3571492', 'n_tokens_mistral': 3817, 'n_tokens_neox': 3575, 'n_words': 1213}