<commit_before>package ops

import (
	"errors"
	"github.com/phzfi/RIC/server/images"
	"github.com/phzfi/RIC/server/logging"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

type dim [2]int
type idToSize map[string]dim

type ImageSource struct {
	roots []string
	sizes idToSize
	mutex *sync.RWMutex
}

func MakeImageSource() ImageSource {
	return ImageSource{
		sizes: make(idToSize),
		mutex: new(sync.RWMutex),
	}
}

func (i ImageSource) LoadImageOp(id string) Operation {
	return loadImageOp{&i, id}
}

// Search root for an image. Returned image should be destroyed by image.Destroy, image.Resized or image.ToBlob or other.
func (i ImageSource) searchRoots(filename string, img images.Image) (err error) {
	if len(i.roots) == 0 {
		logging.Debug("No roots")
		err = os.ErrNotExist
		return
	}
	// Extract requested type/extension and id from filename
	ext := strings.TrimLeft(filepath.Ext(filename), ".")
	id := strings.TrimRight(filename[0:len(filename)-len(ext)], ".")
	// Search requested image from all roots by trial and error
	for _, root := range i.roots {
		// TODO: Fix escape vulnerability (sanitize filename from at least ".." etc)
		// Assume image is stored as .jpg -> change extension to .jpg
		trial := filepath.Join(root, id) + ".jpg"
		err = img.FromFile(trial)
		if err == nil {
			logging.Debug("Found: " + trial)
			break
		}
		logging.Debug("Not found: " + trial)
	}
	return
}

// TODO: This is a temp solution for ImageSize creating too many Images.
// Limit to creating only one at time for finding the image size

func (i ImageSource) ImageSize(fn string) (w int, h int, err error) {
	i.mutex.RLock()
	s, ok := i.sizes[fn]
	i.mutex.RUnlock()

	if ok {
		return s[0], s[1], nil
	}

	image := images.NewImage()
	defer image.Destroy()

	err = i.searchRoots(fn, image)
	if err != nil {
		return
	}

	w = image.GetWidth()
	h = image.GetHeight()

	i.mutex.Lock()
	i.sizes[fn] = dim{w, h}
	i.mutex.Unlock()

	return
}

// A very trivial (and inefficient way to handle roots)
// Can be used for development work, however.
func (i *ImageSource) AddRoot(root string) error {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return err
	}

	logging.Debug("Adding root: " + root + " -> " + abspath)
	for _, path := range i.roots {
		if path == abspath {
			return errors.New("Root is already served")
		}
	}

	i.roots = append(i.roots, abspath)
	return nil
}

// A very trivial (and inefficient way to handle roots)
// Can be used for development work, however.
func (is *ImageSource) RemoveRoot(root string) error {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return err
	}

	for i, path := range is.roots {
		if path == abspath {
			is.roots = append(is.roots[:i], is.roots[i+1:]...)
			return nil
		}
	}

	return errors.New("Root not found")
}
<commit_msg>ImageSize no longer reads the whole file<commit_after>package ops

import (
	"errors"
	"github.com/phzfi/RIC/server/images"
	"github.com/phzfi/RIC/server/logging"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

type dim [2]int
type idToSize map[string]dim

type ImageSource struct {
	roots []string
	sizes idToSize
	mutex *sync.RWMutex
}

func MakeImageSource() ImageSource {
	return ImageSource{
		sizes: make(idToSize),
		mutex: new(sync.RWMutex),
	}
}

func (i ImageSource) LoadImageOp(id string) Operation {
	return loadImageOp{&i, id}
}

// Searches root for an image. If found loads the image to img. Otherwise does nothing and returns an error.
func (i ImageSource) searchRoots(fn string, img images.Image) (err error) {
	return i.searchRootsCustomTrialFunc(fn, img.FromFile)
}


// Searches root for an image. Calls the given trialFunc with the given fn for every root until trialFunc does not return an error. Returns if trialFunc succeeds. returns with error if no trialFunc succeeds.
func (i ImageSource) searchRootsCustomTrialFunc(fn string, trialFunc func (fn string) (err error)) (err error) {
	if len(i.roots) == 0 {
		logging.Debug("No roots")
		err = os.ErrNotExist
		return
	}
	// Extract requested type/extension and id from filename
	ext := strings.TrimLeft(filepath.Ext(fn), ".")
	id := strings.TrimRight(fn[0:len(fn)-len(ext)], ".")
	// Search requested image from all roots by trial and error
	for _, root := range i.roots {
		// TODO: Fix escape vulnerability (sanitize filename from at least ".." etc)
		// Assume image is stored as .jpg -> change extension to .jpg
		trial := filepath.Join(root, id) + ".jpg"
		err = trialFunc(trial)
		if err == nil {
			logging.Debug("Found: " + trial)
			break
		}
		logging.Debug("Not found: " + trial)
	}
	return
}

// Get image size
func (i ImageSource) ImageSize(fn string) (w int, h int, err error) {
	i.mutex.RLock()
	s, ok := i.sizes[fn]
	i.mutex.RUnlock()

	if ok {
		return s[0], s[1], nil
	}

	image := images.NewImage()
	defer image.Destroy()

	err = i.pingRoots(fn, image)
	if err != nil {
		return
	}

	w = image.GetWidth()
	h = image.GetHeight()

	i.mutex.Lock()
	i.sizes[fn] = dim{w, h}
	i.mutex.Unlock()

	return
}

// Searches root for an image. If found, loads only the image metadata to img. Otherwise does nothing and returns an error.
func (i ImageSource) pingRoots(fn string, img images.Image) (err error) {
	return i.searchRootsCustomTrialFunc(fn, img.PingImage)
}

// A very trivial (and inefficient way to handle roots)
// Can be used for development work, however.
func (i *ImageSource) AddRoot(root string) error {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return err
	}

	logging.Debug("Adding root: " + root + " -> " + abspath)
	for _, path := range i.roots {
		if path == abspath {
			return errors.New("Root is already served")
		}
	}

	i.roots = append(i.roots, abspath)
	return nil
}

// A very trivial (and inefficient way to handle roots)
// Can be used for development work, however.
func (is *ImageSource) RemoveRoot(root string) error {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return err
	}

	for i, path := range is.roots {
		if path == abspath {
			is.roots = append(is.roots[:i], is.roots[i+1:]...)
			return nil
		}
	}

	return errors.New("Root not found")
}

{'fraction_non_alphanumeric': 0.13053497942386832, 'fraction_numerical': 0.0019753086419753087, 'mean_word_length': 3.575301204819277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7288186', 'n_tokens_mistral': 2236, 'n_tokens_neox': 2108, 'n_words': 676}