<commit_before>package util

import (
	"encoding/json"
	"fmt"
	"github.com/docker/docker/pkg/parsers"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"time"
)

type ThinImageLayer struct {
	Digest   string `json:"digest"`
	Repo     string `json:"repo,omitempty"`
	Location string `json:"location,omitempty"`
}

type ThinImage struct {
	Version    string           `json:"version"`
	MinVersion string           `json:"min_version,omitempty"`
	Layers     []ThinImageLayer `json:"layers"`
	Comment    string           `json:"comment,omitempty"`
}

func (t *ThinImage) AddLayer(newLayer ThinImageLayer) {
	t.Layers = append([]ThinImageLayer{newLayer}, t.Layers...)
}

func IsThinImageLayer(diffPath string) bool {
	magic_file_path := path.Join(diffPath, ".thin")
	_, err := os.Stat(magic_file_path)

	if err == nil {
		return true
	}

	return false
}

func GetCvmfsLayerPaths(layers []ThinImageLayer, cvmfsMountPath string) []string {
	ret := make([]string, len(layers))

	for i, layer := range layers {
		root := cvmfsMountPath
		// repo := cvmfsDefaultRepo
		repo := layer.Repo
		location := "layers"
		digest := layer.Digest

		ret[i] = path.Join(root, repo, location, digest)
	}

	return ret
}

func AppendCvmfsLayerPaths(oldArray []string, newArray []string) []string {
	l_old := len(oldArray)
	l_new := len(newArray)
	newLen := l_old + l_new - 1
	ret := make([]string, newLen)

	for i := 0; i < l_old-1; i++ {
		ret[i] = oldArray[i]
	}

	for i := range newArray {
		ret[l_old-1+i] = newArray[i]
	}

	return ret
}

func ExpandCvmfsLayerPaths(oldArray []string, newArray []string, i int) (result []string) {
	left := oldArray[:i]
	right := oldArray[i+1:]

	result = append(left, newArray...)
	result = append(result, right...)

	return result
}

func GetNestedLayerIDs(diffPath string) []ThinImageLayer {
	magic_file_path := path.Join(diffPath, ".thin")
	content, _ := ioutil.ReadFile(magic_file_path)

	var thin ThinImage
	json.Unmarshal(content, &thin)

	return thin.Layers
}

func ParseOptions(options []string) (map[string]string, error) {
	m := make(map[string]string)

	for _, v := range options {
		key, value, err := parsers.ParseKeyValueOpt(v)

		if err != nil {
			return nil, err
		}

		m[key] = value<commit_msg>Removed unused code<commit_after>package util

import (
	"encoding/json"
	"fmt"
	"github.com/docker/docker/pkg/parsers"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"time"
)

type ThinImageLayer struct {
	Digest   string `json:"digest"`
	Repo     string `json:"repo,omitempty"`
	Location string `json:"location,omitempty"`
}

type ThinImage struct {
	Version    string           `json:"version"`
	MinVersion string           `json:"min_version,omitempty"`
	Layers     []ThinImageLayer `json:"layers"`
	Comment    string           `json:"comment,omitempty"`
}

func (t *ThinImage) AddLayer(newLayer ThinImageLayer) {
	t.Layers = append([]ThinImageLayer{newLayer}, t.Layers...)
}

func IsThinImageLayer(diffPath string) bool {
	magic_file_path := path.Join(diffPath, ".thin")
	_, err := os.Stat(magic_file_path)

	if err == nil {
		return true
	}

	return false
}

func GetCvmfsLayerPaths(layers []ThinImageLayer, cvmfsMountPath string) []string {
	ret := make([]string, len(layers))

	for i, layer := range layers {
		root := cvmfsMountPath
		// repo := cvmfsDefaultRepo
		repo := layer.Repo
		location := "layers"
		digest := layer.Digest

		ret[i] = path.Join(root, repo, location, digest)
	}

	return ret
}

func ExpandCvmfsLayerPaths(oldArray []string, newArray []string, i int) (result []string) {
	left := oldArray[:i]
	right := oldArray[i+1:]

	result = append(left, newArray...)
	result = append(result, right...)

	return result
}

func GetNestedLayerIDs(diffPath string) []ThinImageLayer {
	magic_file_path := path.Join(diffPath, ".thin")
	content, _ := ioutil.ReadFile(magic_file_path)

	var thin ThinImage
	json.Unmarshal(content, &thin)

	return thin.Layers
}

func ParseOptions(options []string) (map[string]string, error) {
	m := make(map[string]string)

	for _, v := range options {
		key, value, err := parsers.ParseKeyValueOpt(v)

		if err != nil {
			return nil, err
		}

		m[key] = value
{'fraction_non_alphanumeric': 0.1434436126539483, 'fraction_numerical': 0.0014489253803429123, 'mean_word_length': 4.119901112484549, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5485459', 'n_tokens_mistral': 1566, 'n_tokens_neox': 1479, 'n_words': 342}