<commit_before>package scanner

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/image"
	"github.com/Symantec/Dominator/objectserver"
	"os"
	"path"
	"syscall"
)

func loadImageDataBase(baseDir string, objSrv objectserver.ObjectServer) (
	*ImageDataBase, error) {
	fi, err := os.Stat(baseDir)
	if err != nil {
		return nil, errors.New(
			fmt.Sprintf("Cannot stat: %s\t%s\n", baseDir, err))
	}
	if !fi.IsDir() {
		return nil, errors.New(fmt.Sprintf("%s is not a directory\n", baseDir))
	}
	imdb := new(ImageDataBase)
	imdb.baseDir = baseDir
	imdb.imageMap = make(map[string]*image.Image)
	imdb.objectServer = objSrv
	err = imdb.scanDirectory("")
	if err != nil {
		return nil, err
	}
	return imdb, nil
}

func (imdb *ImageDataBase) scanDirectory(dirname string) error {
	file, err := os.Open(path.Join(imdb.baseDir, dirname))
	if err != nil {
		return err
	}
	names, err := file.Readdirnames(-1)
	file.Close()
	for _, name := range names {
		filename := path.Join(dirname, name)
		var stat syscall.Stat_t
		err := syscall.Lstat(path.Join(imdb.baseDir, filename), &stat)
		if err != nil {
			if err == syscall.ENOENT {
				continue
			}
			return err
		}
		if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR {
			err = imdb.scanDirectory(filename)
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFREG {
			err = imdb.loadFile(filename)
		}
		if err != nil {
			if err == syscall.ENOENT {
				continue
			}
			return err
		}
	}
	return nil
}

func (imdb *ImageDataBase) loadFile(filename string) error {
	file, err := os.Open(path.Join(imdb.baseDir, filename))
	if err != nil {
		return err
	}
	defer file.Close()
	decoder := gob.NewDecoder(file)
	var image image.Image
	err = decoder.Decode(&image)
	if err != nil {
		return err
	}
	imdb.imageMap[filename] = &image
	return nil
}
<commit_msg>Rebuild pointers when loading image from persistent store.<commit_after>package scanner

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/image"
	"github.com/Symantec/Dominator/objectserver"
	"os"
	"path"
	"syscall"
)

func loadImageDataBase(baseDir string, objSrv objectserver.ObjectServer) (
	*ImageDataBase, error) {
	fi, err := os.Stat(baseDir)
	if err != nil {
		return nil, errors.New(
			fmt.Sprintf("Cannot stat: %s\t%s\n", baseDir, err))
	}
	if !fi.IsDir() {
		return nil, errors.New(fmt.Sprintf("%s is not a directory\n", baseDir))
	}
	imdb := new(ImageDataBase)
	imdb.baseDir = baseDir
	imdb.imageMap = make(map[string]*image.Image)
	imdb.objectServer = objSrv
	err = imdb.scanDirectory("")
	if err != nil {
		return nil, err
	}
	return imdb, nil
}

func (imdb *ImageDataBase) scanDirectory(dirname string) error {
	file, err := os.Open(path.Join(imdb.baseDir, dirname))
	if err != nil {
		return err
	}
	names, err := file.Readdirnames(-1)
	file.Close()
	for _, name := range names {
		filename := path.Join(dirname, name)
		var stat syscall.Stat_t
		err := syscall.Lstat(path.Join(imdb.baseDir, filename), &stat)
		if err != nil {
			if err == syscall.ENOENT {
				continue
			}
			return err
		}
		if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR {
			err = imdb.scanDirectory(filename)
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFREG {
			err = imdb.loadFile(filename)
		}
		if err != nil {
			if err == syscall.ENOENT {
				continue
			}
			return err
		}
	}
	return nil
}

func (imdb *ImageDataBase) loadFile(filename string) error {
	file, err := os.Open(path.Join(imdb.baseDir, filename))
	if err != nil {
		return err
	}
	defer file.Close()
	decoder := gob.NewDecoder(file)
	var image image.Image
	err = decoder.Decode(&image)
	if err != nil {
		return err
	}
	image.FileSystem.RebuildPointers()
	imdb.imageMap[filename] = &image
	return nil
}

{'fraction_non_alphanumeric': 0.13909066451439334, 'fraction_numerical': 0.0005380683346785041, 'mean_word_length': 3.712294043092522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7630142', 'n_tokens_mistral': 1523, 'n_tokens_neox': 1400, 'n_words': 324}