<commit_before>package main

import "fmt"

// A resource is something that can be deleted. Rating provides an indication
// of how "valuable" it is. A higher value means it should be preserved in
// favour of a resource with a lower rating.
type Resource interface {
	Rating() float64
	Expunge() error
}

// Resources is a type that can be sorted in order to establish in which order
// resources should be expunged.
type Resources []Resource

func (r Resources) Empty() bool {
	return len(r) == 0
}

func (r *Resources) ExpungeNext() error {
	err := (*r)[0].Expunge()
	if err != nil {
		return err
	}
	*r = (*r)[1:]
	return nil
}

// Implement sort.Interface to sort by deletion order.
func (rs Resources) Len() int {
	return len(rs)
}

func (rs Resources) Less(i, j int) bool {
	return rs[i].Rating() < rs[j].Rating()
}

func (rs Resources) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
}

// Note ideally this would run in an independent thread, but since we have one
// job at a time, we can sequence it between task runs. Also it should be
// independent of mounts feature, but let's go with it here as currently that
// is the only feature that uses it.
func runGarbageCollection(r Resources) error {
	currentFreeSpace, err := freeDiskSpaceBytes(TaskUser.HomeDir)
	if err != nil {
		return fmt.Errorf("Could not calculate free disk space in dir %v due to error %#v", TaskUser.HomeDir, err)
	}
	requiredFreeSpace := uint64(config.RequiredDiskSpaceMegabytes * 1024 * 1024)
	for currentFreeSpace < requiredFreeSpace {
		// need to free up space
		if r.Empty() {
			break
		}
		err = r.ExpungeNext()
		if err != nil {
			return err
		}
		currentFreeSpace, err = freeDiskSpaceBytes(TaskUser.HomeDir)
		if err != nil {
			return err
		}
	}
	if currentFreeSpace < requiredFreeSpace {
		return fmt.Errorf("Not able to free up enough disk space - require %v bytes, but only have %v bytes - and nothing left to delete.", requiredFreeSpace, currentFreeSpace)
	}
	return nil
}<commit_msg>Fix for incorrect calculations on 32 bit systems<commit_after>package main

import "fmt"

// A resource is something that can be deleted. Rating provides an indication
// of how "valuable" it is. A higher value means it should be preserved in
// favour of a resource with a lower rating.
type Resource interface {
	Rating() float64
	Expunge() error
}

// Resources is a type that can be sorted in order to establish in which order
// resources should be expunged.
type Resources []Resource

func (r Resources) Empty() bool {
	return len(r) == 0
}

func (r *Resources) ExpungeNext() error {
	err := (*r)[0].Expunge()
	if err != nil {
		return err
	}
	*r = (*r)[1:]
	return nil
}

// Implement sort.Interface to sort by deletion order.
func (rs Resources) Len() int {
	return len(rs)
}

func (rs Resources) Less(i, j int) bool {
	return rs[i].Rating() < rs[j].Rating()
}

func (rs Resources) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
}

// Note ideally this would run in an independent thread, but since we have one
// job at a time, we can sequence it between task runs. Also it should be
// independent of mounts feature, but let's go with it here as currently that
// is the only feature that uses it.
func runGarbageCollection(r Resources) error {
	currentFreeSpace, err := freeDiskSpaceBytes(TaskUser.HomeDir)
	if err != nil {
		return fmt.Errorf("Could not calculate free disk space in dir %v due to error %#v", TaskUser.HomeDir, err)
	}
	requiredFreeSpace := uint64(config.RequiredDiskSpaceMegabytes) * 1024 * 1024
	for currentFreeSpace < requiredFreeSpace {
		// need to free up space
		if r.Empty() {
			break
		}
		err = r.ExpungeNext()
		if err != nil {
			return err
		}
		currentFreeSpace, err = freeDiskSpaceBytes(TaskUser.HomeDir)
		if err != nil {
			return err
		}
	}
	if currentFreeSpace < requiredFreeSpace {
		return fmt.Errorf("Not able to free up enough disk space - require %v bytes, but only have %v bytes - and nothing left to delete.", requiredFreeSpace, currentFreeSpace)
	}
	return nil
}
{'fraction_non_alphanumeric': 0.10252563140785197, 'fraction_numerical': 0.008002000500125032, 'mean_word_length': 3.6948356807511735, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7348600', 'n_tokens_mistral': 1308, 'n_tokens_neox': 1228, 'n_words': 525}