<commit_before>package bundle

import(
  "github.com/TykTechnologies/goverify"
  "github.com/TykTechnologies/tykcommon"
  "io"
  "io/ioutil"
  "encoding/json"
  "encoding/base64"
  "archive/zip"
  "crypto/md5"
  "bufio"
  "errors"
  "bytes"
  "fmt"
  "os"
)

var bundleOutput, privKey string
var forceInsecure *bool

const (
	defaultBundleOutput = "bundle.zip"
)

func init() {
}

// Bundle will handle the bundle command calls.
func Bundle(command string, thisBundleOutput string, thisPrivKey string, thisForceInsecure *bool) (err error) {
  bundleOutput = thisBundleOutput
  privKey = thisPrivKey
  forceInsecure = thisForceInsecure
	switch command {
	case "build":
		var manifestPath = "./manifest.json"
		if _, err := os.Stat(manifestPath); err == nil {
			var manifestData []byte
			manifestData, err = ioutil.ReadFile(manifestPath)

			var manifest tykcommon.BundleManifest
			err = json.Unmarshal(manifestData, &manifest)

			if err != nil {
				fmt.Println("Couldn't parse manifest file!")
				break
			}

			err = bundleValidateManifest(&manifest)

			if err != nil {
				fmt.Println("Bundle validation error:")
				fmt.Println(err)
				break
			}

			// The manifest is valid, we should do the checksum and sign step at this point.
			bundleBuild(&manifest)

		} else {
			err = errors.New("Manifest file doesn't exist.")
		}
	default:
		err = errors.New("Invalid command.")
	}
	return err
}

// bundleValidateManifest will validate the manifest file before building a bundle.
func bundleValidateManifest(manifest *tykcommon.BundleManifest) (err error) {
	// Validate manifest file list:
	for _, file := range manifest.FileList {
		if _, statErr := os.Stat(file); statErr != nil {
			err = errors.New("Referencing a nonexistent file: " + file)
			break
		}
	}

	// The custom middleware block must specify at least one hook:
	var definedHooks int
	definedHooks = len(manifest.CustomMiddleware.Pre) + len(manifest.CustomMiddleware.Post) + len(manifest.CustomMiddleware.PostKeyAuth)

	// We should count the auth check middleware (single), if it's present:
	if manifest.CustomMiddleware.AuthCheck.Name != "" {
		definedHooks++
	}

	if definedHooks == 0 {
		err = errors.New("No hooks defined!")
		return err
	}

	// The custom middleware block must specify a driver:
	if manifest.CustomMiddleware.Driver == "" {
		err = errors.New("No driver specified!")
		return err
	}

	return err
}

// bundleBuild will build and generate a bundle file.
func bundleBuild(manifest *tykcommon.BundleManifest) (err error) {
	var useSignature bool

	if bundleOutput == "" {
		fmt.Println("No output specified, using bundle.zip")
		bundleOutput = defaultBundleOutput
	}

	if privKey != "" {
		fmt.Println("The bundle will be signed.")
		useSignature = true
	}

	var bundleData bytes.Buffer

	for _, file := range manifest.FileList {
		var data []byte
		data, err = ioutil.ReadFile(file)

		if err != nil {
			fmt.Println("*** Error: ", err)
			return err
		}

		bundleData.Write(data)
	}

	// Update the manifest file:
	manifest.Checksum = fmt.Sprintf("%x", md5.Sum(bundleData.Bytes()))

	// If a private key is specified, sign the data:
	if useSignature {
		var signer goverify.Signer
		signer, err = goverify.LoadPrivateKeyFromFile(privKey)

		if err != nil {
			// Error: Couldn't read the private key
			return err
		}
		var signed []byte
		signed, err = signer.Sign(bundleData.Bytes())

		if err != nil {
			// Error: Couldn't sign the data.
			return err
		}

		manifest.Signature = base64.StdEncoding.EncodeToString(signed)
	} else {
		if *forceInsecure == false {
			fmt.Print("The bundle will be unsigned, type \"y\" to confirm: ")
			reader := bufio.NewReader(os.Stdin)
			text, _ := reader.ReadString('\n')
			if text != "y\n" {
				fmt.Println("Aborting")
				os.Exit(1)
			}
		}
	}

	var newManifestData []byte
	newManifestData, err = json.Marshal(&manifest)

	// Write the bundle file:
	buf := new(bytes.Buffer)
	bundleWriter := zip.NewWriter(buf)

	for _, file := range manifest.FileList {
		var outputFile io.Writer
		outputFile, err = bundleWriter.Create(file)
		if err != nil {
			return err
		}
		var data []byte
		data, err = ioutil.ReadFile(file)

		_, err = outputFile.Write(data)

		if err != nil {
			return err
		}
	}

	// Write manifest file:
	var newManifest io.Writer
	newManifest, err = bundleWriter.Create("manifest.json")
	_, err = newManifest.Write(newManifestData)

	err = bundleWriter.Close()
	err = ioutil.WriteFile(bundleOutput, buf.Bytes(), 0755)

	return err
}
<commit_msg>Exposing bundle functions.<commit_after>package bundle

import(
  "github.com/TykTechnologies/goverify"
  "github.com/TykTechnologies/tykcommon"
  "io"
  "io/ioutil"
  "encoding/json"
  "encoding/base64"
  "archive/zip"
  "crypto/md5"
  "bufio"
  "errors"
  "bytes"
  "fmt"
  "os"
)

var bundleOutput, privKey string
var forceInsecure *bool

const (
	defaultBundleOutput = "bundle.zip"
)

func init() {
}

// Bundle will handle the bundle command calls.
func Bundle(command string, thisBundleOutput string, thisPrivKey string, thisForceInsecure *bool) (err error) {
  bundleOutput = thisBundleOutput
  privKey = thisPrivKey
  forceInsecure = thisForceInsecure

	switch command {
	case "build":
		var manifestPath = "./manifest.json"
		if _, err = os.Stat(manifestPath); err == nil {
			var manifestData []byte
			manifestData, err = ioutil.ReadFile(manifestPath)

			var manifest tykcommon.BundleManifest
			err = json.Unmarshal(manifestData, &manifest)

			if err != nil {
				fmt.Println("Couldn't parse manifest file!")
				break
			}

			err = BundleValidateManifest(&manifest)

			if err != nil {
				fmt.Println("Bundle validation error:")
				fmt.Println(err)
				break
			}

			// The manifest is valid, we should do the checksum and sign step at this point.
			bundleBuild(&manifest)

		} else {
			err = errors.New("Manifest file doesn't exist.")
		}
	default:
		err = errors.New("Invalid command.")
	}
	return err
}

// BundleValidateManifest will validate the manifest file before building a bundle.
func BundleValidateManifest(manifest *tykcommon.BundleManifest) (err error) {
	// Validate manifest file list:
	for _, file := range manifest.FileList {
		if _, statErr := os.Stat(file); statErr != nil {
			err = errors.New("Referencing a nonexistent file: " + file)
			break
		}
	}

  // The file list references a nonexistent file:
  if err != nil {
    return err
  }

	// The custom middleware block must specify at least one hook:
	var definedHooks int
	definedHooks = len(manifest.CustomMiddleware.Pre) + len(manifest.CustomMiddleware.Post) + len(manifest.CustomMiddleware.PostKeyAuth)

	// We should count the auth check middleware (single), if it's present:
	if manifest.CustomMiddleware.AuthCheck.Name != "" {
		definedHooks++
	}

	if definedHooks == 0 {
		err = errors.New("No hooks defined!")
		return err
	}

	// The custom middleware block must specify a driver:
	if manifest.CustomMiddleware.Driver == "" {
		err = errors.New("No driver specified!")
		return err
	}

	return err
}

// bundleBuild will build and generate a bundle file.
func bundleBuild(manifest *tykcommon.BundleManifest) (err error) {
	var useSignature bool

	if bundleOutput == "" {
		fmt.Println("No output specified, using bundle.zip")
		bundleOutput = defaultBundleOutput
	}

	if privKey != "" {
		fmt.Println("The bundle will be signed.")
		useSignature = true
	}

	var bundleData bytes.Buffer

	for _, file := range manifest.FileList {
		var data []byte
		data, err = ioutil.ReadFile(file)

		if err != nil {
			fmt.Println("*** Error: ", err)
			return err
		}

		bundleData.Write(data)
	}

	// Update the manifest file:
	manifest.Checksum = fmt.Sprintf("%x", md5.Sum(bundleData.Bytes()))

	// If a private key is specified, sign the data:
	if useSignature {
		var signer goverify.Signer
		signer, err = goverify.LoadPrivateKeyFromFile(privKey)

		if err != nil {
			// Error: Couldn't read the private key
			return err
		}
		var signed []byte
		signed, err = signer.Sign(bundleData.Bytes())

		if err != nil {
			// Error: Couldn't sign the data.
			return err
		}

		manifest.Signature = base64.StdEncoding.EncodeToString(signed)
	} else {
		if *forceInsecure == false {
			fmt.Print("The bundle will be unsigned, type \"y\" to confirm: ")
			reader := bufio.NewReader(os.Stdin)
			text, _ := reader.ReadString('\n')
			if text != "y\n" {
				fmt.Println("Aborting")
				os.Exit(1)
			}
		}
	}

	var newManifestData []byte
	newManifestData, err = json.Marshal(&manifest)

	// Write the bundle file:
	buf := new(bytes.Buffer)
	bundleWriter := zip.NewWriter(buf)

	for _, file := range manifest.FileList {
		var outputFile io.Writer
		outputFile, err = bundleWriter.Create(file)
		if err != nil {
			return err
		}
		var data []byte
		data, err = ioutil.ReadFile(file)

		_, err = outputFile.Write(data)

		if err != nil {
			return err
		}
	}

	// Write manifest file:
	var newManifest io.Writer
	newManifest, err = bundleWriter.Create("manifest.json")
	_, err = newManifest.Write(newManifestData)

	err = bundleWriter.Close()
	err = ioutil.WriteFile(bundleOutput, buf.Bytes(), 0755)

	return err
}

{'fraction_non_alphanumeric': 0.11633527408546633, 'fraction_numerical': 0.002636493463693288, 'mean_word_length': 3.561122244488978, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3773970', 'n_tokens_mistral': 3258, 'n_tokens_neox': 2933, 'n_words': 828}