<commit_before>package abide

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
)

var (
	args         *arguments
	allSnapshots snapshots
)

var (
	// SnapshotsDir is the directory snapshots will be read to & written from
	// relative directories are resolved to present-working-directory of the executing process
	SnapshotsDir = "__snapshots__"
	// snapshotsLoaded
	snapshotsLoaded = sync.Once{}
)

const (
	// snapshotExt is the file extension to use for a collection of snapshot records
	snapshotExt = ".snapshot"
	// snapshotSeparator deliniates records in the snapshots, not externally settable
	snapshotSeparator = "/* snapshot: "
)

func init() {
	// Get arguments
	args = getArguments()
}

// Cleanup is an optional method which will execute cleanup operations
// affiliated with abide testing, such as pruning snapshots.
func Cleanup() error {
	for _, s := range allSnapshots {
		if !s.evaluated && args.shouldUpdate && !args.singleRun {
			s.shouldRemove = true
			fmt.Printf("Removing unused snapshot `%s`\n", s.id)
		}
	}

	return allSnapshots.save()
}

// snapshotID represents the unique identifier for a snapshot.
type snapshotID string

// isValid verifies whether the snapshotID is valid. An
// identifier is considered invalid if it is already in use
// or it is malformed.
func (s *snapshotID) isValid() bool {
	return true
}

// snapshot represents the expected value of a test, identified by an id.
type snapshot struct {
	id           snapshotID
	value        string
	path         string
	evaluated    bool
	shouldRemove bool
}

// snapshots represents a map of snapshots by id.
type snapshots map[snapshotID]*snapshot

// save writes all snapshots to their designated files.
func (s snapshots) save() error {
	snapshotsByPath := map[string][]*snapshot{}
	for _, snap := range s {
		_, ok := snapshotsByPath[snap.path]
		if !ok {
			snapshotsByPath[snap.path] = []*snapshot{}
		}
		snapshotsByPath[snap.path] = append(snapshotsByPath[snap.path], snap)
	}

	for path, snaps := range snapshotsByPath {
		if path == "" {
			continue
		}

		snapshotMap := snapshots{}
		for _, snap := range snaps {
			if snap.shouldRemove {
				continue
			}
			snapshotMap[snap.id] = snap
		}
		data, err := encode(snapshotMap)
		if err != nil {
			return err
		}

		err = ioutil.WriteFile(path, data, os.ModePerm)
		if err != nil {
			return err
		}
	}

	return nil
}

// decode decides a slice of bytes to retrieve a Snapshots object.
func decode(data []byte) (snapshots, error) {
	snaps := make(snapshots)

	snapshotsStr := strings.Split(string(data), snapshotSeparator)
	for _, s := range snapshotsStr {
		if s == "" {
			continue
		}

		components := strings.SplitAfterN(s, "\n", 2)
		id := snapshotID(strings.TrimSuffix(components[0], " */\n"))
		val := strings.TrimSpace(components[1])
		snaps[id] = &snapshot{
			id:    id,<commit_msg>Don't create snapshot files with the executable bit set<commit_after>package abide

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
)

var (
	args         *arguments
	allSnapshots snapshots
)

var (
	// SnapshotsDir is the directory snapshots will be read to & written from
	// relative directories are resolved to present-working-directory of the executing process
	SnapshotsDir = "__snapshots__"
	// snapshotsLoaded
	snapshotsLoaded = sync.Once{}
)

const (
	// snapshotExt is the file extension to use for a collection of snapshot records
	snapshotExt = ".snapshot"
	// snapshotSeparator deliniates records in the snapshots, not externally settable
	snapshotSeparator = "/* snapshot: "
)

func init() {
	// Get arguments
	args = getArguments()
}

// Cleanup is an optional method which will execute cleanup operations
// affiliated with abide testing, such as pruning snapshots.
func Cleanup() error {
	for _, s := range allSnapshots {
		if !s.evaluated && args.shouldUpdate && !args.singleRun {
			s.shouldRemove = true
			fmt.Printf("Removing unused snapshot `%s`\n", s.id)
		}
	}

	return allSnapshots.save()
}

// snapshotID represents the unique identifier for a snapshot.
type snapshotID string

// isValid verifies whether the snapshotID is valid. An
// identifier is considered invalid if it is already in use
// or it is malformed.
func (s *snapshotID) isValid() bool {
	return true
}

// snapshot represents the expected value of a test, identified by an id.
type snapshot struct {
	id           snapshotID
	value        string
	path         string
	evaluated    bool
	shouldRemove bool
}

// snapshots represents a map of snapshots by id.
type snapshots map[snapshotID]*snapshot

// save writes all snapshots to their designated files.
func (s snapshots) save() error {
	snapshotsByPath := map[string][]*snapshot{}
	for _, snap := range s {
		_, ok := snapshotsByPath[snap.path]
		if !ok {
			snapshotsByPath[snap.path] = []*snapshot{}
		}
		snapshotsByPath[snap.path] = append(snapshotsByPath[snap.path], snap)
	}

	for path, snaps := range snapshotsByPath {
		if path == "" {
			continue
		}

		snapshotMap := snapshots{}
		for _, snap := range snaps {
			if snap.shouldRemove {
				continue
			}
			snapshotMap[snap.id] = snap
		}
		data, err := encode(snapshotMap)
		if err != nil {
			return err
		}

		err = ioutil.WriteFile(path, data, 0666)
		if err != nil {
			return err
		}
	}

	return nil
}

// decode decides a slice of bytes to retrieve a Snapshots object.
func decode(data []byte) (snapshots, error) {
	snaps := make(snapshots)

	snapshotsStr := strings.Split(string(data), snapshotSeparator)
	for _, s := range snapshotsStr {
		if s == "" {
			continue
		}

		components := strings.SplitAfterN(s, "\n", 2)
		id := snapshotID(strings.TrimSuffix(components[0], " */\n"))
		val := strings.TrimSpace(components[1])
		snaps[id] = &snapshot{
			id:    id,
{'fraction_non_alphanumeric': 0.10753058762708943, 'fraction_numerical': 0.0017232465965879718, 'mean_word_length': 3.4646153846153847, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4340090', 'n_tokens_mistral': 1945, 'n_tokens_neox': 1839, 'n_words': 601}