<commit_before>package main

import (
	"fmt"
	"log"
	"os/exec"
	"regexp"
	"strings"
	"time"
)

var (
	pt             = fmt.Printf
	snapshotFormat = regexp.MustCompile(`[a-zA-Z0-9]+@autosnap-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{2}`)
	snapshotFreqs  = [...]struct {
		Age  time.Duration
		Freq time.Duration
	}{
		{time.Hour * 24 * 30, time.Hour * 24},
		{time.Hour * 24 * 14, time.Hour * 6},
		{time.Hour * 24 * 7, time.Hour},
		{0, time.Minute},
	}
)

func process(name string) {
	// take snapshot
	now := time.Now()
	run("zfs", "snapshot", fmt.Sprintf("%s@autosnap-%04d-%02d-%02d-%02d-%02d-%02d",
		name, now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()))

	// delete unchanged
	out := run("zfs", "list", "-d", "1", "-t", "snapshot", "-H", "-o", "name,used", "-p", name)
	lines := strings.Split(out, "\n")
	var toDel []string
	for i := 1; i < len(lines); i++ {
		line := strings.TrimSpace(lines[i])
		if len(line) == 0 {
			break
		}
		parts := strings.SplitN(line, "\t", 2)
		if parts[1] != "0" {
			continue
		}
		lastLine := strings.TrimSpace(lines[i-1])
		lastParts := strings.SplitN(lastLine, "\t", 2)
		out = run("zfs", "diff", lastParts[0], parts[0])<commit_msg>adjust<commit_after>package main

import (
	"fmt"
	"log"
	"os/exec"
	"regexp"
	"strings"
	"time"
)

var (
	pt             = fmt.Printf
	snapshotFormat = regexp.MustCompile(`[a-zA-Z0-9]+@autosnap-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{2}`)
	snapshotFreqs  = [...]struct {
		Age  time.Duration
		Freq time.Duration
	}{
		{time.Hour * 24 * 30, time.Hour * 24},
		{time.Hour * 24 * 20, time.Hour * 6},
		{time.Hour * 24 * 10, time.Hour},
		{0, time.Minute},
	}
)

func process(name string) {
	// take snapshot
	now := time.Now()
	run("zfs", "snapshot", fmt.Sprintf("%s@autosnap-%04d-%02d-%02d-%02d-%02d-%02d",
		name, now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()))

	// delete unchanged
	out := run("zfs", "list", "-d", "1", "-t", "snapshot", "-H", "-o", "name,used", "-p", name)
	lines := strings.Split(out, "\n")
	var toDel []string
	for i := 1; i < len(lines); i++ {
		line := strings.TrimSpace(lines[i])
		if len(line) == 0 {
			break
		}
		parts := strings.SplitN(line, "\t", 2)
		if parts[1] != "0" {
			continue
		}
		lastLine := strings.TrimSpace(lines[i-1])
		lastParts := strings.SplitN(lastLine, "\t", 2)
		out = run("zfs", "diff", lastParts[0], parts[0])
{'fraction_non_alphanumeric': 0.2502071251035626, 'fraction_numerical': 0.04763877381938691, 'mean_word_length': 3.3357271095152603, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4676714', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1042, 'n_words': 207}